{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T02:24:01.114479Z",
     "start_time": "2020-02-17T02:23:57.651863Z"
    }
   },
   "outputs": [],
   "source": [
    "import importlib\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import seaborn as sns\n",
    "from lightgbm import LGBMClassifier\n",
    "from matplotlib import style\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn import metrics\n",
    "\n",
    "import model_evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 0. Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T02:24:05.719476Z",
     "start_time": "2020-02-17T02:24:03.473482Z"
    }
   },
   "outputs": [],
   "source": [
    "# Load selected features\n",
    "with open('../data/feature/feature_selection.pkl', 'rb') as f:\n",
    "    feature_col = pickle.load(f)\n",
    "    \n",
    "# Form training set\n",
    "file_train = pd.read_csv('../resources/File_train.csv')\n",
    "df_train = pd.DataFrame()\n",
    "for file in file_train['file']:\n",
    "    df_train = df_train.append(pd.read_csv(f'../data/feature/{file}.csv'), ignore_index=True)\n",
    "\n",
    "# Form testing set\n",
    "file_test = pd.read_csv('../resources/File_test.csv')\n",
    "df_test = pd.DataFrame()\n",
    "for file in file_test['file']:\n",
    "    df_test = df_test.append(pd.read_csv(f'../data/feature/{file}.csv'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-17T02:39:46.976695Z",
     "start_time": "2020-02-17T02:39:43.392250Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARYAAADQCAYAAADCi05SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOy9eZSk6VXe+bvvt0RE7mtlLVlLd1dXqxe1tm6JRccjzM7B6AwGBnkWBoMNY8DMDGiAMYOx8JkRjD0MHhgxAssCjzE2NmANIxCCQUeAQVJ3I6nVe1XXkrXnvkbEt7x3/rhvZGZVZWZFVmVVZVbHc06czPjiW96MjO/GXZ77XFFVOuiggw52Eu5eL6CDDjq4/9AxLB100MGOo2NYOuiggx1Hx7B00EEHO46OYemggw52HB3D0kEHHew4Ooalgw7ewBCRj4jIVRH50iavi4j8MxE5KSJfFJG3t3PejmHpoIM3Nj4KfMMWr38j8HB4/F3gQ+2ctGNYOujgDQxV/TQws8Uu7wV+XQ1/CQyIyIGbnTe+2Q4i8gzwL4DfUNXZdhd8LzAyMqLHjh2718vooIPbwrPPPjulqqPt7PsNX/9unZra/LZ89rkXXwAa6zZ9WFU/vI3lHAIm1j0/H7Zd2uqgmxoW4DuB7wY+t87I/KHuwl6AY8eO8cwzz9zrZXTQwW1BRM62u+/U5Dk+8+n/YdPX494faqjqU7eznA223fTev2kopKonVfUfACeA3wA+ApwTkX8kIkPbXmYHHXSwY1CgVL/pYwdwHji87vk4cPFmB7XjsSAiT2JeyzcB/x74V8C7gf8PeOt2V9pBBx3sFFIkGr+TF/gY8IMi8pvAu4B5Vd0yDIL2cizPAnPAPwd+XFWb4aXPiMhX3saCO9jDUG2i5RRoHaQfiYYQie71st5wUG1SFOdu+XgR+dfAe4ARETkP/EMgsXPrLwMfxxyKk8AK5mDcFO14LN+uqq9v9IKqfusWC/4I8M3AVVV9Imz7aeDvAJNht/9RVT++wbHfAPwCEAG/qqofbGOdHdwlqF9C8y9gjngEeh71A5A8gUhbTnAHOwSRFIkP33zHTaCq77vJ6wr8wHbP2065+XtFZKD1REQGReQft3HcR9m4Pv7zqvrW8NjIqETAL2H188eA94nIY21cr4O7BC1eBRLEDSCuF4lGwM+j5ZV7vbQ3HFQz8uLcpo97hXYMyzeq6lzrSSg5f9PNDmqjPr4Z3gmcVNXXVTUDfhOrpXewC6CagV9EXNe1L7ge8JMbH9TBHYMCXnXTx71CO35rJCKVVm5FRGpA5Tau+YMi8l8BzwA/sgE3ZqO6+bs2O5mI/F2MEciRI0duY1kdtAcH4lD1iKz7XtICpLqjV/LlDJRnwS+CG0Dio4jrv2YfVQ86h/pFoIZEg4gkO7qO3QyRlCjefZ/7djyW/xv4YxH5HhH528AngV+7xet9CHgIqyRdAv7pBvtsq26uqh9W1adU9anR0bY4RR3cBkRicAfArzqxqJagK0h8U0JmW1At8MVVyL8AFOAGgSaa/RXq59ftl6P582j2PJTnoXgJzZ5Dtb4j69gL8JqRFRObPu4VbuqxqOrPicjzwFdjN/3PqOonbuViqroahIvIrwC/t8Fut1Q37+DuQeJjKDlaXgURQCA+jrjbozWpNtHidQup8pPBAzoIoiA1EEWLs0j6pO1fXoLyioVhUkFcBfWLaH4aSd84abnyXi9gA7SVwlfV3wd+/3YvJiIH1tXA/1Ngo47KzwEPi8gDwAWM+fu3bvfaHewcRGIkeRSNj4HmIFVE0ts6p6pH8y+BNkD7oZgBWYLiDESHIOoDd9TCIoKXlH0WygUIlSiNRsEdBD+FavmGKH+LpMS7MBRqh8fyrcDPAvswj0WwKlTfTY7bqD7+HhF5KxbanAG+L+x7ECsrf5OqFiLyg8AnsHLzR1T1hVv78zq4kxCpmSexE9B58MtINGT8GGmA9AE5SAp+CfxZSB6y3csJMypRr72uat4LFZCEjSPq+w9ecxrF+Xu9jBvQjsfyc8DfUNWXtnPiTerj/3yTfS+yrtIUytA3lKI7uI+h2Zot8JMQHYbyElBYYpgY9CJEX4WqQnEB4gcsuesSC8lcD5SnofKeaxPL9zFEUpJoD3oswJXtGpUOOtg2pAtWe1sUXDdw2AyHZvY8eghxvWGfEtww0AR/FVRCZaob2YWhwZ2C14z6HvVYnhGRfwP8LtCi86Oqv33HVtXBGw/SA26fJW/Vg58yY5I+acliPw9urW1AozHwU0g8juqoGR9dhPjEG6rcDFDuwrCvHcPSh/UIfN26bQp0DMsehqpHy2kLOyRGorEbOCLtngedD6FMFaQPkVv5oDfMMPg85FOmQRNwvbZO14XED67uLfFRNJ9HyxmQyLwVN4JEB2/h2nsXIimVXeihtVNubqvpqIO9A1WPFi9DedWSr+rR8iIan8DF7d+Yqhmav2AGwTZYeJK86aY9Q6pqLQD+ghmFchpcL5KaCqKWK1B83vgp0TDIUdZ/XEWqkLzdDAt1C4Hc4BuiErQeXnOW8z0YConICYzYNqaqTwQJhW9R1Xb6hTrYjdB5KK9aj09rk3ZBeQqNRtouHWtxDnT5Gv6KltNoeemmjXFanoXiNLhQXCzOQDSMul5ALAmrYmXmaD+UZ1AyJDmxeg6RGIn3tf1n29/ZCAZtEVyfeWpyO0TyewuRlOou9FjaSZ3/CvATQA6gql/EuCUd7FGon7MS7TqIROZxtMlaVdVQtamhfgXV3F5wfeC35jOqZlCeAzdsN7XE4KoW/hTnoZwFGuB6QSSwfYehvHRbrFr1y2j2LJQTQB3Ks3ueqet9xlJ+YdPHvUI7OZYuVf3sdXFzcYfW08FdQQV0M75me6GEqofyMugyiAMEdWPgRgIbd6uDrQawWhIul+xcftkIcsRWJRLAHQn7CorYsbfIndHiLODWKktSQ/0CWpxDkkdu6Zz3HoLX3Vdab8ewTInIQ4R+HRH5Nm4ipNvB7oZEw2j5OqoNy1VAqLr0I66nvZP4y4FfEpmXoh7Ki2YcKk9veIiqgi6hugya23MyKylHx0BPWsKWEsrXoPLXQHrXjkVvr9HRT4W+o3WQbiinYI8aFicJXcmhe72MG9COYfkB4MPAm0TkAnAa+C/u6Ko6uKMQqUDyJFq8jJazGG9k6Jr8xXqoNoOnkK4aIsoJSI7bTz9vXztagiuR6MZmRNUGmr8YKPliN3M5v5ZjEYxV68aBEoqUohB8kZFUBNEFiMbXrr9NWLiTgV+AaH31qwC3d3MspeYs5Jfv9TJuQDtVodeBrxGRbsCptkoAHexliOuD5GlsMoTbMIFplZvTISfhrHoUHYTowUBaG4T4YQuHNMcUDTfu0dH8ZdCmeUt+GahAcQqogSsgegDcfiCnyOpMvJgwdXURdIau/i6OPfEkPUMP3nDem0FV0eKUVZfKJfAvo+UhSB4AxAxdvHcbFp2kdCe7r8TeTlXop657DoCqfuAOramDuwT7X26er9DiIuQvBH5KxXp3ygv2ezQKftYMlFj4pH4uGIdwvDatHOyXoLyCxIfQch7Kk1gI1Y/le5ZB9tnN7xc5/fk55qcX6B8dRNLHadYdr/7VMm9+d0Fa3V6zo5ZXzTC6kZAA7obiJOQZxIcgfgiJbk1uQ7Vu75HOWbk7Gm8/lNwhlJozvxc9FmB53e9VTMe2Q/G/z6FaQv7nFjpIDWvOjyF+EIrzSOVtaDZvoZSklnSVKhKbYrwvp4OeijPvJn8BLSat1E0TKMA3wA3YTZ8/D1qn0ehmbipjYOwxI76VE1S7T5A1MmauzLH/6PbKy/iLVlZuJZTjA6gbBj8DyTtx7ta6stWvoPlf2ROpgZ+1Mnb6FsQNbH3wDmNPJm9V9RoxJhH5J9hIgA7uY9g3/QxE++wGBytFlxfNEFBF0rej5STokjFuo2FEUnw5C/XfDYS5Kmhk+RSumqFyDmQMXBpyLHUgguQERd1B3AtR+OYvF1AtcJEja+S38JeU3MCqkCTki26dTKflBUDMYwOQNHgwryNpW3PTdwROEnqSnRHY2knciqR6F7D9YLeDvQU/CdEBY9XKWnmW4pKJOokAKRJfW5Hw5RSs/BYUE5YkLavhXKNQTgba/ijIFYjeZB6Ln8RCowGqPVXELeNLj3OyWrrOs4LewVsIM9x+y+VEw2vbdNHo/7fD0tUZK4mvg0gN9TN3VQum1IK5bPeJmLeTY3keVqUhI2AU6ORX7nsIRCNQNs3bkNj6eCSC+NiGR6jm0Pgz8IWpvuVnQugTQ1w1wSYtAu+lz3I2moMvIT0G5WVi18vhR3o488IC1VqTKBmkMbdC/3AvfcM9qzKYRshTYwlLz6b9SRLtR/209RtJFCpXVSS53e/GLvPg1rUu2JoqtMc73Rk4Sejdox7LN6/7vcBkFG5KkNtkrtD/CvwNIANOAd+9fgLAumPPAIuYH1vc5uzZDm4BEh9Cs2lTbWMepLBH/ADObazxpeWseQPlLBRfYvUG0wUzKJWvgOQEUFgSuLyAyR7kUL8cQq6SsdFeam8bZfLyGIUe4cDDowztH8TJIpq9BMXrFpJJNxofNwOYPL5hZUskhuTNmOD2EmuC27c3/0jiw2j2eVQjRCpmVPw8xI/cYhPmraHUnNnb9FhuNsdLRI5gOtcDYZ8f32h0z3q08+5eX17uW//GqepmIz4+Cvwi8Ovrtn0S+ImgEvezWKvAj21y/Fep6lQb6+tgB6FahErOguVYygvBw4ghfnRThqqqB38FigUonjNvRErAQyHW8KgppMchf8W8FJ0CBPR8qCYNQ3wA1NM30k3/+LtWE6GqTbT5RSjOQfkqULPj8ybwDpRXVrVwr4eIAxm6bU3ea87pBtDkzVCeDOXz2CQbov03PXan4W/DQ1o3x+trMb3pz4nIx1T1xXW7/STwb1X1Q2HG18eBY1udtx3D8hwmbj2L0ZgGgNYkJGWTfIuqflpEjl237Q/XPf1L4NvauH4HdwlGYvsi+JCkLWetRBuPm3HxpYUu1/UZqTZNrzZ7DfLnAsM1A60BNctv+ASYDuKmA+DOGRnOL0HZMAKeP225i2Qc/BxaTq8ZlnLWcjT+Eki/JYUBistW7vV6DZP4bsBFw6gbwtro4nuiWuckoS+9LWO2OscLIMxofi+w3rAoJp8C0M8ODYX/A+BjLddHRL4R+BpV/ZH2174h/jbwbzZ5TYE/FBEF/i9V/fBmJ+nMFdo5aHHWDIfrgTKH+GCQRGgg7iCqc2h5BXHXfpdofgrKOQsFopHAR3EgHmTZCHVRAfERiJ8EfcXCo3gEshegXAGZDZm8MiR8L0P8KF5qiOu2delMoPTnlivxddtengZ5s3VWkwNVxPXcFU5JK4l9r1D6gpnG1a12GRGRZ9Y9//B191M7c7x+GrsffwjoBr7mZutqx7A8rarf33qiqr8vIj/TxnGbQkT+AZav+Veb7PKVqnpRRPYBnxSRl8NkxRsQ3qQPAzz11FP3bvTbHoe2xKjdAMYzaVVkuqGYsgbA/Bzkp/F+CuIjiBsB1Dgo+cnQlFjBPla59f0443iY1agar0S8qcNp0/bVKas4SRm6o5ehmITiNZASLUqrwPjMvBV/do3t6zNLEvtZyF+zPBARGh9F4weR+OH7Wv82kpiBdGyrXaZukqNsZ47X+4CPquo/FZEvB/6liDyhuqolegPabUL8SWxwmWJ9QtNtHLchROS7sKTuV4eB0zcgiGujqldF5Hcwd21Dw9LBTiLC/sWp/a6FPdcpKHKr8JTzULwIJGjyOMgoNP/CwhtxwIL13vgSWLT946NQ/VokfcT4MXRbmJSfsfAGAd+EKFRtykvhHIsgNcRVTbk/OmBlbD8fiHtV48KoggpIDvFhI+SV07ZGGUDiLW+8PY1CC6ay20pFtjPH63sIc9hV9S/E4s0RYFNXqR1T/j6sxPw74TEatm0bIfv8Y5hQ1Mom+3SLGHEi9Cd9HRvPH+pgByEilkvxs8bBiI7YjV1eCOHGFGSfD02EkXk3jf8IjX9nDFrJsC+/wryeaNRYuvGT1u2cPBa6m4Hsi5B9GsrnzYho04ySHDZj5kZBglB2/hJaXDZvKD8fqk7z1lZAN8g4RI+Esm8g0EkKeKsy+d1Hd99ZmJzEZo82sDrHS0zh6zu5kQB7DhtYiIg8ijHwtxzU3Q7zdgb4YRHpUdWldlYaFrDRXKGfwAr9nwyVpb9U1e9fP1cIGAN+J7weA7+hqn/Q7nU7uHVINB7Yo5dAVwKVvwd00vp7pCd4EkEK0fWbFxMfCSXlEsp6qAYlEA9D/CbMI1mA/CVo/AX4V+11CaQ1STBeyLIZNL0MpGEEyBXIXjdeTDlpzYrxPtNpkRJkMCR+lRu+J1tSC62nfgEtzgT1uF4kPrbGnN2jiCRmMN1mm8M6bDbHS0Q+ADyjqh8DfgT4FRH577A39L/eLNpooR2C3FcAvwr0AEdE5C3A96nq37vJgm9prlDITr/lZuvqYOfR8lQsXBGrCHmF4iULbaQw+j4OyOwjpmI3sBwAmYeogc0CioGazXiWbvNu8ufATwAeM1gzQL/lXLQw48N5y8vIgCWB81Nh/9Q8GeehzOz6kXlYtgZnYRGYgRPBRoRYGKR+Hs3+ijxPcXEXkdbR7DlI33ZLIuK7BYUWTDZuj5Wx0RwvVf2pdb+/CHzlds7ZTo7l54GvJ7hHqvoFEflr27lIB3sHWrxmfTRhCJayZCVd5ozIRmzlZOkCuRySrjVr6qNhfUE0LR9CYXR6xcIYMozzGGG9rUV4DAQj02UhkabmoRTnoThroVV8yFoEXBqU5nLzPLQObh+4w+a5lOftetE4xONIZIZlceoVzr44y8oiuEgYOzrAgQeqROtmQe9FxBIzdBsey51Cu7ObJ65jE+7GOdQd3AJ8OQ3lGbtZpWt1Vs8qyovAcviPL1m4QmRhiALJg6FnZgnoBhdZIlaXoCzNQMgApvvisTyMhucNIAlJ4i7zLoqLlj+RljcyD16giEAbEB2060f7zdhoYepvWobzZDYqxA2tUv3ryw1e/uwrVLsGGdiX4kvPpdOzlEU/Rx/J7tZbfUeQa8HV5pbpjnuCdpK3EyEcUhFJReRH6cgm3BfwxVXIvwioiTZpHYpTqLe8uqq3hK0qRIl5Aa4PSC38SB4JItiz5jUkB2wfqWAzlyt2bn8JmMMMSZdto2n70MTI3R6TWKjbfuWEGSw3CC2tW5+bB+TnrCJULlp4Vpwy46gXIRrFxUcR17tKrZ+5OItE3VS6LC3gIkf/SBdXz02SF9136+2+QxAUt+njXqEdj+X7sT6CQ1hp6g8xucoO9jCMt3LadG5bTNpoEC377SZ1j4VQw+b94AZAZ4FBqwC5B8O8HzHyWzkNRZCw1BnQrqB5chh8D3AWMx5g4U8r2SoYqTMBLoY5zMOB69IHehCYXjtURi0HkzwatFaGVicfquZQvIy6d14zwqS+VCepHjT+ixML9cgR6pR+96mvbQexxAyntyZUdSexpWEJfQT/par+53dpPR3cNZRAE6vor0N8HIpX16YkqoDrMg9Bq+ZRqANyY84SQV4P+Q2bM4RvYh0gBfAARBUoe0B7jROzylSN7fiWil25bKxfv7CufA34EPpEfdZQ6PrBDYG/es04VZEE9SX4BRNz0gXQBr1DyszllFr3CTNG5QKlryDpcSrduy8/sR3kvuBqc/e11G1pWFS1FJH3YgncDu4TqKo1zmkD9fPXVUUUkncg8QHrz1klpF3BbvSlwKRNIQ9l6XLeBrNLr+U6KDHxJhf0Ww5hpLhWOJTadfBh3yXMCCXBu5kJpeJB46JEPRDvt1J0fAQkQcTdQA9d+/vyoEg3AzgGhzKuVOeZnz5EV99DFHlJY7nBQ289RhTt/cmJ3u8+ZnE7odCfi8gvYn09qzKVqvrcHVtVB3cMNhb1xZAUbVjoEI9D9BB20+dIfJS1vEYWKjNjRk4jBQZBz0C+FISoi7VKTVQBp3YcDsoF265N1uIZH64lWAjkwyMOud0+SwJTN56K9JsXk/SD1q3DWqyCpJpfGwpJZIljP4dEI2g5Q+yu8siTM0xdmmd25ihdfUd58Mkj9A313rX3/U4hdjGj1ZGb73iX0Y5h+Yrwc724kwJ/feeX08GdhjUaLiFuGKWGsWhPW7iRPoTEj4PENjEQB/FDoQN5wm5YSSAeCxT6QROmlmrguFyGMngZVLDE7LnQ2dwfnidYsrYl6aOY1xJ6jKI+y6PoEvhzlquhABkx1bf00VVSm8aPmmH03s4jEcSPWo+R6zOvrDwN0kNSO8iBo8scOB5BJLhk7xsVsFDoUv2WO2zuGDY1LCLyw6r6C8D/pKp/dhfX1MEdgrb6cKTPDEzZ+kDWQJpIcsI0a4uzoXRbAX8BWAljT6+CO2QGpJVHKSeDrm2MGYhZy8XQxJjf9fB70zqffUjuIhjnMgn7VIIXMhTyMGqEuPiRkBBegeT4NUxZF42g7p2BWEcQzU7xxWt2vJ80er/EgTTnQm7mIqpHb1vsaTcgkZixPeaxfDdWDfpnwN1TB+7gjsEMyxSUL1kPTXQIoiHAW8KzOGvEtixoquSnTNKgvIDlQaLQK7SATTDsBlwgzrUqPcuYRzIMDIZ9q1hoVQYezLq+Hhwm8RPOL0AZpC2JwrUxo5A9D7X3XPM3iaRmsNYjOmBD5jVj9SPulyAes9yMb3lJe9+w5FpyqbGZ1tq9w1bv7EtBInJURL64brsAqqp7l674RkV5Eli2b3LXY6XkcgWkD6KjUJxDy8uYAamHBsQMpGmhD/MYHyXCPgYzmDGJWcuTtF4rrdtYY7smaTBKLQ8mGA5b2NrxZWZEONVQGj4QhKKaULyI+rfedLyGRIdQvxiMaPhbXS+4MZvq6KrcSw2VnYQq+E3FC+4dNjUsqvo+EdmPNSd9y91bUgd3Ar5cDBWaR4NuSsPyJeVVSEatL6g4Zd/+8bj19mgdaw6cwej4jjVKPmEbrFZ0rtm2xOpsZ8DKz12sddvHNkZEqsZ30SEsD+PM0DkxHotLrWXANSEaQ4sLSHoTwyIxJI/b1MbsOVufGzJvSQtI3nxXdWnvJBK390IhVPUynYbAPQ/185A/A/nLod/moJWQXbfR7d1+YDk8r2I3dy00+11mna45ZihizOtYvQJrhgfM+8jD87Gwf8NekhgYCDT/zNizbgiSQ3b9+GHIXoTybHBipsGtGL9GhsHPoqo3NQwigkRDaPU/saZKPwtSQ6KxG5Tl1M/YRENMskGisWvyL63JAOBMzW4XIfcFF+t7KxTq4D6A+mXTsSUxPoh0GXFMYkCMiKZ146BE+6F82bqR/TRIPfBJbCaz3emK5UPi8Fg/sGG9T15g4cYspmZYCdcbsIRsMRkSw2FcK5H1FkX1QGzLrTKk2HW8abMQDaP5Mybs3Yb0pEgSZh8d2vB1X5wzT026rapUnET9FCRPIBJZL1Xxsv39Cur6keRNG04EuBdIXMz+2vDNd7zL6BiW+xzaIra5Acup+KVAZFuy8IcDUHmbGZDihaDqNg3565hxcFgIA2teSIzd8S2j0mosvB4F0Msqg5YEmIfsCubxNFhVnvMLwXgs2Rpdv71OxXqO8i9B+hYLk/wCmn0BTd+BrHpCvdseEqaamWfkhtaOjSqBdTyHSs1mV7veNa6MX0Dzl5F0dzjymS+5tDJ7r5dxA+6oYdlkttAQRrY7BpwBvkNVb3hngoTlT4an/1hVf+1OrvW+hV8JzYAe6DWGrAa9lfgdSHICAG1+ZlUK0rRMMqyi02370hKNbhmVIOZEjoVHG42ayoApuy6pnc8La1UhCcauFhqfg9atlhaukVoCmB4LRYqLoWRcMSPkp1E3EOxWYrOFthBuUr9i3ggl4oZoaRXdYJAkRXU+8HbctW0Drg8tZ1C/vGvConIXKj23I/R0Ang/cHT9/qraDkHuo9w4W+jHgT9W1Q+KyI+H59fMFgrG5x8CT2Gf4mfDrJPdZ5p3O9wgFFMmsOQXIRqEshZumgFE0jB/eda0TKTL1PmjISv7tkrA2sqtuPBYxv41m3krLZSYEUlZ46wU4Vw1wJvBiIbNG2mJZLNgHdNkVqGSRVAPRbiut/CtNdHQRpA8D+m7NuSn+OKqCVaJVa20OGvJY+9v7PHXwtbWkoi4HgLXhn33DomLOFjbuXlJO4V2PJbfAn4Z+BW2qcOy0WwhbGbJe8LvvwZ8ihuHln098MnWMDQR+SQm5vuvt3P9DkCi0TA98KIxZimNcu8eBX8G1f1Wmm1+ASOlVawZkAQzBnnIs7SqPd3h92r4WXDjx6IV+rQMTmD4ssxarqYL62oOeRhfN+9EDkA0z2r1ycdQvma0/qQ/GMYlayb0l9auKBX7O3TBSHbroJpD+Uro5I7DNjWujuu2nijXHzguy+alRMOgFbQ8j3lc685FxPVzm+8Vcl9yYeWGYaL3HO0YlkJVP7SD1xxT1UsAqnopjPi4HhvNOtkw+9aZK7Q1RFI0PgisAJklKaNjiOuxaohfDtMOI0x0KbIbrpy051INid7gOVBiIVKCGZkhLNxZL4fccgFaBqdlqFpGqBUKLYR9vTFkaQCzZhjKS6BnQw4mARfbWjSweKUHdB7VYp2HIubVXA9dBlXErX3cRQSVOOSZSvCXrTXA9SPx4/a+MQjRfuP2tMJJLcM4193RvJhIxIHa4L1exg3YitLfMvv/j4j8PUyhf7XGuMVo1Z1AO7NOWuvozBW6GVwPREPXEMtMMb+VgPUQvxmaf2yKcX4eu/G7IRoz1bZiBRte2aLjtwyMsFaCbuVZNvJgctaqSq08zTLmtfQDFXDOCHtMWpgSj5uhcHGQnwzyCtEg+NdAe2gZKtXSLuM26gFygcl7PRSkiosPovoA4K/RcbHJBY9AtM/GzkqCuFHE7Q5vBSx5e355b3ksz7IWRIPlWVrYdLRqG7giIgeCt3KAjWeTnGctXAKbdfKpW7zeGx4S7UPLc6jWEamhvoDyVMhllEZQk75QcTlnCVxNsF6gPKizNTBDUmPNOLTkJbuwcCEPz1sGJsZCplYoVcE8lxZTV4FRiLuDcPe8JXB9HeRQ0GYpTXRqdbRHML0TYBEAACAASURBVGrRmFWO/BIqK2aA4hMbl4HFyuzrE64W0oBEQ+G0G98KIsJOz31+I2Ar5u0Dd+iaHwO+C/hg+PkfNtjnE8D/LCItH+/rsNEhHdwCRKqQPIkWr6LFtJVYiSE5gfFHXjf+ihZB8sCzSobTlqfRtOc41oxHizTXxDwPYVVicjUn0zIgErYHPguE82QQf5mR47K/BNkH8TJEwasRherXQPG5UCoPa0gfD+Vn+wiLG9y0SiPiIH4cLV60UrIELyt+7K7Oer4TSFzMwb0UCrUgIt+6weZ54HlV3XJo7CazhT4I/FsR+R5sENK3h32fAr5fVb9XVWfCGNfPhVN94A6HXvc9xPVB8g40moSsuGY6oCZPQPMzmBBTiumutLySRawLuZfVRsFV49LqD6qzZjhahgjWSHT18LwIr/dgnswoJA9YyEMe8h0FcMhyGtG45VTi/ZC+z+QQ/KzxcNwgyM1nNKtm65i3g5AcNa9GundNnuR2kPmCiT0WCrXwPcCXA38Snr8HC7ZPiMgHVPVfbnbgJrOFIExVu27fZ4DvXff8I8BH2lhfB22iRYPXVWEkD34u3LAhpJFKUNpfwKLUludRY81raQk0rVeBW+HaZsQqa9WgLOxbAxkJhqEC0aNQfZcJSeksRoYLxsVVTJuFLkQi80bStwaC2vOhNC5ooWh8BImO3UDztxL05010ylUDCe8iJG+5L4wKQOpixrtvz2MJE0p/Aftn/aqqfnCDfb4DGw6vwBdU9W9tdc52DIsHHlXVK+ECY8CHsIn0nwY2NSwd7B4Ytf8lY5IW59H4CaCwMmw5aZ6BzgZtlcCQXRVmqprurS8w76PFwG2FPa1k7fX0/pb3UrWf7kFIHrJQxNeh9lU2PqQ8Y4lkEXAluCMW6khqFZ2WsJN6+xuksiqnqeqDwt2QlaTX/83lJdAcicKNJ7Uw6fEkkq4pgaiWNhvaX7UEbbT/ph3UuwWZLzm/NH/Lxwdd618CvhbLbX4ucMZeXLfPw1gq4itVdXaTSu41aMewHGsZlYCrwIkQruSbHdTB7oH6FbTxqTBSI/TEND9hcpKuFmQl5zDN2RaTNii3EVuHsZ+131fZtmE+8mqYA2sl5Va1qTVkvgJUIT5hFR5qkD5qj/wFy/VowwxcedF6l/wyuNx6glqVGl0BbSBurTdGxKGSon76xomGrcFq62DJ65lVSUszVi9YAtt1g19Gy0to/Agu3gMK/q3i3q3jncDJMIEUEflNjGv24rp9/g7wSy2C6s1SINCeYflTEfk9jCgH8DeBT4eB7bsvuLsPoVpYuVPnsQ7d0W01wWkxYQ1/0aglO6VmM5HLM6CjlrAlDonQBqbg1sMq61RzLEypYbosJWsGY/2numWUWq9HmPEJw8xYgOidWJk5N8kGbRoZjT4r6+ojJtIUH7Suaz+NLy+FnMpmZd71xcv1qJiRCobJSuz1wHVp/W1z4GeRlliUgGoNylNoNHoNnb9dlGVJcyXDRY5q151tVkyjiEPdW3pXIyLyzLrnHw4UjRY24oy967pznAAQkT/H3rifvtk89XYMyw9gxuQrsf/erwP/PgyF/qo2ju/gNqCaW07BLxhBjAItz1qeoI3uXjvJNGuGA2zucb9xPnQyaNK2JCMrrPUEVYA6psTWUtavYLyTxfC4Hq2wKF33PND+sxfM66h8DSTjoTq1lutQbZpXVZ4P+ZAZS+hG+4JgU6/18fiVVS6JSRrkiLtRk0TicTT7PKph2mJx2sIddwDNvwTJw6ifWzU8q8dJtGaEtmlYZq/Oc+b5sxS5vQ/9o70ce+IoaWX7BqodZGXJ+aUtv9+nVPWpLV5vhzMWAw9j+dVxzNl4QlU3vfBNDUswIP8uPDq4y9DyMviltW9UQmhTnETSt25+3OoYjsRyD5qt5THwlkfRZbt5XL8JK/kVzFj0YkS4zI6nCbiQYA05F2pcy7Zdj9YQsjzs79Ye5ZTJFMT7jMUqkeVJUMhPmnekkSWURcK1SzMqHpPT1Fnz4FqXih9GNiDGiRtE48egeNUkF0RM7yU6ALpsIZCENocb3kDYbo9ufanOyedep6u/i+7Ujl2cWeLMl85y4h3Ht3WudpG6iPGu28oHnQcOr3s+DlzcYJ+/VCP/nBaRVzBD8zk2wVbM2z9T1XeLSKv2uPoSZm82byPtYOfgJy32XwdxXaifRjW7hikK5u5rORFU9UurrriDlgAtJy0cUm+hEClmMK6ECvEKllsJg91Xv8xaCVq1Y6Qb9BKbkKHD9qWw70Ag1+UhDItBFyB/BZLHbOZQS59WZzGjlZt3Fg+EjusUa5rsBhpI+pSdQz24ni3DQheP2Z+mdYj2rRHhpBf1MxClZmdb5EFVC4/c0LYZtjOX5nCRI0nXbqveoR7mJxdorDTvSFiU+ZKJ20jeYsbhYRF5ALgAfCdwfcXnd4H3AR8VkREsNHp9q5NuRZB7d/h5f8xJ2LNIgku+tsW8EcdGnbdanjePwA0hLrLwojgF6buh+WdQ/yO70b2zsCR5IJSW5zAvpFXt6bZrSwY6iCnJZVgOJmO192hTFPbQSjhn0Jn1K+Y5+QUgwsXHUDdooUmeB47drHkuRagK6QXzNsoliA6HUCUNNP12vAoFV72RXauKEK+RB8uZQMcZReKH2jjvtciznCjeeHhYWWyrf/euQVULEflBjJQaAR9R1RdE5APAM6r6sfDa14nIi9g3zPtVdcuZI235eqEk1dIYbC3o3K39KR1sBxIfMlEjrazF/n4OovFruBiqpeUL8hfAjay+JlJBpYDiPJBB+mbzXvILkM/ZjR7tMwNSNDCDkYafEhK3VzE6vw/bNzZqa2iJQ7X6iBKj4JfzmBZuDuSQv4SPjyLRPkBBnJHYuGjjWosFcAW4A3a8LEN5Cd9smMERgNgqONHmKmriuo3vsk7SUlvNiq4LkRok76AV8l3vBbaL/tF+rpydpKtvzdPJs4Iojqh235kkbuoiDvfcHo9FVT8OfPy6bT+17ncF/vvwaAvtMG9/CGPMXmGNUqlAR6X/NlBfqjN1YYbmSpP+0T4GxwaIkxv/HeKG0PgElK+vNQ5GY2FaYSv0OQ/ZMxb6+IvgplB3MJR2xfIpxYuAMxarX7ZErdSgmAvkOFjtBYrGwtiNBdbKyS0R7dYaNxvrGQwJFSzPEgMzlpBFgSELY5qXIbL5RFr767b2aNwSrPmpYITmwFeAC1ZB6vqbdh5dWafDkkPxAuqeNgOxAcT1mrB2ecFU4cC8tfiB1WPM4Nwevb9/pJfBsQFmr8xRqaWUhacsSx5++0N3bJRrVpZMLOy+4mw7HssPA4/czPXpoH0szCzyymdP4iJHnMbMXJ7j6rkpHnn6+IbGxcWH0GgfLWX9Vn+LcTC+BM0/XysFl9OQzwPP2RRD1x3CiQiiLrthi5eN46EZlmC4EvIXQ0A95F8WsG/wFnclY41B21KMC2NRV4lxrZtnndqcq5lCHA7Tv52EYjaIOHnI/6NxV+JRbLD8jOVVWAJyMzYSvB+9amuRtW9o46I4tJxE4s1lMyQ+Dm7I6P0iSHTC8j87COccx9/2APNTi8xemSOtJAwfHKTWs7HB2wmkUcThnt1H5mvHsExgNMwOdgCqypkvTVDrrZJWzeWu9VSZu7rA1IVp9h8b2/A4keSG0qeWU1ayle5QNbls+Zj8ZUuS+gXzUGQMxBtztjxvBioawZi3QVtFgyaKRpi+yvrRHS2PptVk2OKBtGj9Lb5KK5lbhmPq4LuMg+LrYZ/QOqCLlrspp03MiTCpsHjRXpPE1lnOWmmaRmiOnIf4+rBiM2nM9e+fQDQcODN3Ds45Bvf1M7iv/+Y77wCysmRicffdnltVhVrx1OvAp0Tk/+VaPZb/7Q6v7b5E3sxXw5/1qPWacdnMsGwIPxm+zWfNKynnzKBEIQyRmt2srhq4Gt5kKiMXyrqt0GjOQihtjSNteSOtPqCW/OR6w6LrfjrMQxGgIM+7KL0gkuLSmIQUmMQMQGhi1EZQx09DqfuMhWR+BePUNLFwahGKV4J0ZdVCmBt0bZuI230dvncNu1CFaCuPpVUNOhceKffL+Lh7CBc5EMF7j3NreYoyL+ju21550yQEUjMe5Yx5HcU0liztg/hICHcU8onQ4BcqQlEPpE/YjZy/iN3wFbtxV6UQQnl5NexZj/VVDk9rrtDyQo2sCQUHgSaRm6Kre5ZqtYLxWlqPLjtGBsxoFJfNq4n6wjyjZjhvqAzRDyxB+g2Bg3IllLEdJG/aVlhjucgGELWdqDUi3hIgID0mxbALYMnbu+MdbQdblZv/0d1cyBsFcRKz78gIV85cpX+0DxGhyAua9Zx9R7Y30U6i/dZoJwNQPGNGwYcwxo1b34+fN14G80YMcwdt1Gr0kCnD6cvBO+jCys3TrBkV2LqkvB5KnsHcdA+VWhPhMtXaMhI1KHOlSOvELmHNQPUADSPnSQxuDHQe6IUojHjVwIqVYUifAhmBymOQP4eFTlWWFyKuTJwiK2BwbD/Dh4a2ZLmqn0OLV6zjWRR1+5D4oS2p++rn0PxF1vqgqpA8tiEp724j83ssFOrgzmH8xAHUeyYnpgEhThzH33aMnoHtjZMQN4BGR6H4BMQPhC7f1G6a8qzR11kGapA8DsmIyTpmYmEIeQh71EIqvYBVgVp5k+1xL1RLunom8b5KpVZHJCeOS8rcoWXTOpfpB3pDJXo8KMHtB1ZARwJJrmZeDCZoTfo4VJ4y76s8D3iIxpifWuG1v1ogTpSkcp4LJ5XJ89M8+mUPk6Q3GgrVurVHSBcSdYfS/SSal0j6+CZ/UxYYulVEWl3WDUuap09vqjx316CCjVTZXbjr74qIPILNFWrhQeCnVPV/X7fPezBludNh02+r6gfu2iLvMKIo4tjjRzh4/ABlXpDWUqIoollvUuQl1a4KUdxeeVJcP+qOWiOhDEL2PMil4L1k2L94DqJmkD1oYsPLnJWhdQW7gc+xKv8IbNeogHHYKl0lqsuIOCIn5HlKnqUk1VbC1wH9FoZVvxrILIdSnjUvikrIt5haPpUnoPqVgLfQL38ddApVYWriCiP7Y0p/BF+skPb0sDi9xNT5GQ48eGOuSkubp9Ri6prs5CBaTq0yb2+AnwctELeWMBapBhmKUyiL4JvG6o3HNy153ymkUcThvj0UConIz6rqj4nIt6vqb22233ahqq8Abw3XiDAa8e9ssOufquo379R1dyPSSgKVhCIvOPn8aWYuzyIIEjmOPjbO6KGbVzC0vArl6dBQOG0PGcGIaIQSbQzNZ4PyfRHItbk1+UUHA4N1ibW8xi0i5HgFK+AUWUJRCFGkREkXa8ngkK5r/on9jHohfjwQ/yoQPW3J5PiAeSvaCJWsB6D8IoiSNxs0lzN6BqrE6Tnq5TFAqHRXmJ9a2NCwWL5pg4+8YDya8MWvqqwsrJBnBdVqnTTZwCPwU2Z04pY271U0m4b07bdMsLsVZGXJxPzeCoW+SUR+EhN42THDch2+Gjilqmfv0Pn3BCZeucjclXn6RyznUhYlr3/+DNWuCr2DW8su4i+FMKYa8gb9IQQSiEooujFuYz2wb3ug7GNVTrJFnKM12uPWkYToo8ggEqX0ShwrtZ4Ityr8VIW4F+KjkH8Bmxl9DNwi+NSIe24AkqctvPNLIXlbQPac5Ye0IHL7QJfx2QpRxeGSN6NiVbe+oU3eMxkAvWjvwep7GDRnAjcoa+ac+vxpFmeXzMi7OkcenGLkcD/izItUn4G/AsnTa31K0m+5mPIqEo/f1vu4Hew5jwX4A4zQ0C0irTkPq6PvdqgJ8TvZfAjZl4vIF7BOyx9V1Rc22mmvzxXKs5yp89P0jfSu0s2jOKLSlTI5Mb2lYcEvGhOXqt2kfjLE3PNhDEaLgt8yGE0sAdmqusRYDsZzu0alhSQBFycUWUGtNydOHW5VQLvPcjzRoK0rGrRKVvb5MO5jxAxkdChwdrxViYoZI9SVS4CNWI0kZ2h/hcnzSwyMVEAGyRo5RVYwMr6xpyfREOot9LEyfWleTPwmvBecU86/coHl+ToDoy2Ful7Ovz5Htec8PYP9sDqF8SDirg970tADdfew5zwWVX0/8H4R+Q+q+t6dvrCYv/gtbKy+/xxwVFWXROSbsO7KhzdZ556eK+RLj3NN4qhAifDeBJZcHJFnWwv0KVhPDTFEJ4ASyoZxVMpJzENoYtWeILS0OqKjZWS2jzy36avqIU7NZMTJ2muzVx3NZg9dPQ1EhN4BT5IEoSgGjADnz1vOx1+xsMLtN0/KHTQPRnqNkSsCbtD6pChNIpMUXA/7jo0g8TxzVy6wWM+odnkeefr4pmV7kQiSx4MM5SRIlZWlHiZemWdp7iLihJnLcxx5dHzdMUJcPcaVCwU9I6GkHfdB8Tyq5XXauc2QiL7L2IWf+nb0WN4bdG6fDps+o6qTO3DtbwSeu072snXNhXW/f1xE/k8RGVHVqR247q6BqpLEFxgYeg3xjrRiXbvN/AT1xYzhA4MszS2SpEskaR2oIdHwuhjeWydwuWAlY62Dn4DVIe4tA9JS0G/9vsJWn8Y8h2YdyhIqVZvEkawzHC0pRHNd7fcy2KjlJUgrJS4SRHrI8wEmryTsH1/GaZ+FOeJtknn2RatiSb+xc1mxMSQ8xWqyVsvVEATXTe6HaS5dJIpXqHR3MTZeZeTQ2/Hxk6TV7hsEta+HSBQmFIxRX6rz8mdfJqmm9I/20VhpcOXsJD0D3YwcWpsjJE4oiypuXcuA1yNQvG5D6YkDxyVCom0QHHcAezEUAkBEvh34J9jAMAH+DxF5v6rervDT+9gkDBKR/cAVVVUReSd2V9x/vUo6B+VZ9h97mDMvXaS+AmnSpNH4IssLh5l46RznZQJ0mUMPjXLggW4oU0ieNNX6cgaKK1jHRQraYrdWIe6HIsbkDurkuc0KimOPiBmEogDnjDbSQp5DHuxPtWqC/a3tTiwpK2Lb1QEKhYc4BMpaRqSVkihu0KgfptQx8JfJ6ktEcQPxF4ndPnB1YAG0y7yQcsKqWq4fiqumcCe9UHwJzV+HeIxLZ5p86c8miF2DLO+hNpDxlncfon/sOEnSppreOlw9P41EjlqP5VeqXVVGx4eZePUSQ/sHjMwIrCzUefDJa8NsiY6gJEH3ZtE6yuOj25IM3QlkRcn5uT0UCq3DTwJPtwR0RWQU+CNuQ1FORLowVfDvW7ft+wFU9ZeBbwP+GxFpycJ/Z2jdvq9gsX6Vrv4ejr/1GAtTS2TNDOYn6eoWBvblSAle9jNxcplqzyBDY4IWp4yXUp4ycln8WOhQFmz2cS/Qj3KBlUUlywtqtQIRT+7BaygKqXkihUBaMUNTFmY04sgMyHp4NeNShgJKEbSfXBQ8llZ1RaHIFXQRLZTmyiJXGo6kouR5le7eKfpHFkmixDyV6IAdrAuWN/JXLbeii6ESdJql2Re5eupVDhyuoQhTlwe5cnqZZ/Ml/tp3HL0lSnh9oU6ldu2Rhx4+wNzkK8xcmiWtpZSlZ3Csn6ED17YMiAgSH4T44DVyDHcbaRRxuH8PeiyAu06Ve5rNe+bbgmqLPHHNtl9e9/svAr94O9fYG1gTgU6rKSPjQ6gqz/3RZfpG+hCdAFfDidDTX+Xy6VmGDxwJw9wXzZjoImRXreLhSyjmgRXy4gpzU8tkDejpK1lZ8nT3msdRX4K0CkkYl4w3jwTsFHFyo1EpcpifiakvOtKap6e3IKlCfdleT1KIY8sdlyqUJSgriGZcmUgYOyT0DTZx2QCLi1XyfJED40FjV+sQHwcds9J5+jagbmJVIqhEzF/6Pbp6V8jyg+TNIfpHPd4dYeoiLM81SMe2L3nQO9jDpdNXV5tBAaLYcfwtRzn25iMUWUFXXxe9gz1bGo57ZVQgaN7O392EcTtox7D8gYh8grWw5T/jOlGYDm4NEo2i5QVU13IDvlyhLCu4uAvKiBZpLYodzXoeelbE9Emyz1gCVEuj5vslLH/SYGG2iqqnuzfkO4hQX5I1zXA4t0rbAGeniWIzENe7ho268Ornu8iaQpIozYZQq5U88FiDJDXiShoE+7WAelNwrkQkI8sd3b1Q6ytR71E/Q7U2TGNxkUazi2p1Cli2KpDUjI3rBsE31m5YXWal3sPSwjgu6aMo+nHRMqP7TlPUFS1HUf/Ytucrjx4e5urEFIuzS3T11sibBSuLdY6/7QFGDu6dWc1yG9SjO4V2krfvD2NW3419Fj+sqhsR2jrYLmTAVOizZ1G/DK6Kiw4zeOhxlufqdPWMmnYKjqXpFcYe2G95legQND5ljE+JAzu0jpWOl8hzJYkL4liJXYFLIKpZFSdJgCh4Khp4YRJGJYP12F23zIunU3wJA8NWku7ug+krEROvx+w/koM3g1QGxUzn7Vu8WReqtQa18Qykh2ajRhTVUZmmKGoUOg6xD6EbED1ipLioD/yFtRDDz9MzMML85EnLa+CpVmeIZIpqzzG6e8RU9uLjuPgw7SKtpjz6ZSe4cnaS2StzVLurHH1snP6RvSPnnEYR43s0FEJVfxv47Tu8ljcc1AdltWg/xF1kWZMLL89w+Yxw+fQKXb3Kvv2XUb9ItavK2IEl0OOQnbEB6oSkp85hnk2L5CbESW6JVzUhORErsrjIDEy7zbnew+xkQt/gtTyXvoGSxZmUww/mFKXlZyihUrPEsBPA9dJcKdAyJ4qqePWkVY9ETWrdMWl1HuQoJI8AhTGAo2HE7UOjeSgvoqFJceSAMDc5xMRrObWeqzidQr3n0acGSOKLpjRXvIRGYzdlvprq3hT486RRxuHjIxx508N3PfG6E8jKvZu87eAOwBcXoPkpC1+kxuxUxGc/mXH1/DzV2kVc1IOvn2LilYzx4z307xvEuxPmweSv2ElcBNoL5SJWGbJsahwrvlyr6FyP7XT8i5h34/211aOyFEQU76G3z/YrnRmYKDJluWY9I89iirxKfckxONpgYXaULI/ZN+5JpWnM4aKOST0cAnfQvJT4OCpdQSKzh6Ta5Pjbn6RnZIGl6Zfp7a8xfGCAWm+CJX6XVkfHyk0mGK4JjveAVMBfQvNpSN56V+n4O4E0ihgf2KMeSwc7C/XzNjcHIBql2VBe/txplmeb7D9cwfkpmiuvceHULNXulJX5GabPvMbKzDMcfqRCd5e3fEQxgZHfWgxbQ6scvBMQgbHDTS6+XqF/pFwtVS8tRIw/WLfQSiwvU5YWEvnCsbRYZWXBsTCrDI3C0IFBymKQan83Y4M5XbU5bIxIDtpt1it5ExSvodEwIrFR4wM93hdXSOLTHH64Cx48ZqX0aMgYtCJAbEa6vGhTFK9/z1VZnF1ieX6RWL5E/+gYSRSMiAyg5QxaTiPxgZ154+4SsrLk/GzHY+mA0DgoFcst6DzzU0JRpERyAacDQEGz4VlZ8gzuq6NlwtBIHZFzzJ6DrgcrSHoMNMXm+9xYFUiStUpPC3F8Y7UH1jgtm2HscE6z4Zi+lCJOcQ6OncgYf6hYPa4M6gvOQVYo0CRKSmo9Eb6IyZoLjD90jOnLyqkXIpwMM3rIM3KgxPkLwIhJaroeND68KhbegovH0GgUyFDvYelDNiK2jMzIui4zKLpyQ/nXe8+pL5xl5tIszmX4bJKkmvPIU4fo6m31+lSsCZK9ZVis0fReL+JGtEOQexj4X4DHWCdjrqoP3sF17TpkjYzJiWlmJ+epdlXYf2zftvVTVqGFyUj6KSjPoJnat2dZR+lHiFlaKEnTiKK5QrUGcbpAkjRZmksosmUSOYelWYOq2Wor1xqSzbWLrsHNqqVRBA+8qcmBoxlF5kgrnrSq1xijKLbH4pxxZJyU9PULUdrF1MUe4rryynOvsrK0j56BflT2cfrlGZYXmjzwWFcYQRJitPwlNNp3gwSBqbZVUb1ohkTDAnTB6uTRI+AqVqJWDZIQyszljJlLswyO9VvTYd5FfVk488JVHvuyVrI3u2Ew3F5AGkcc3qOh0L/Axn/8PDar+bu5sXBwXyNrZLzwF69S5iW1ngpLs0u8eGmWh9/x0K2JJquzUCg+BPIYfcOXiOQkA6MRs3PDdHUvgV6gWVdqXTl9A4sIy3jFxlc4YXlxnqyxTKVWUqnqWlVnC1zvwUD7xgegWlOobd2sWJZC1oyJ4phSU7LlmN6+ggtnekgqjrED86azG0UMjpRMXowZO1qlq7/F6amAr6P1P0Hj/eAOING+1Z4cVW9cl/gx8KeBrlAZW7JEePLVYQRtq/yuTJ2do9p1yEaF+CkoZ6mlc8zNjNCs7yethHlCd5mOvxPIi5LzM7cXConINwC/gNG2f1VVP7jJft+GKR08rarPbLRPC+0Ylpqq/rGISJA3+GkR+VPM2LwhMDkxTZmX9A0bbTypJMRpzrmXzjMQ5CW3h8Xgti8BMbXeYQ4/mnLmpWlq2s3MZU+RefaNZxx7k5BWK4gs4fD0DxdMXiioL2X09MPyoqOxknL04TrJFkWNjYxKa3uSbBw6bRsKIjErixGVqkPFE0Ul9UaFSm2FOA7SCcV58F2I9CBM0VxJ6OpbAhkGSY1564aNhVteQf0Rk4IUseO1ROJhtBRL7mojtEPtQ6IRNH8W1CORcVFctIRmr0BWDToqy0YwzM8gzQZUvhxJju/JqlASRYwP3rrHEjSRfgljwp8HPiciH1PVF6/brxf4+8Bn2jlvO4alIeaDvhZGMV4A9m1n8Xsdc1MLN0yyS6sJ85N18qzYUmN1Q2gTouNA3YyLwv5jPQwMTLO0OIG6AXq7j7M0c5Jzr5YU+RJLCwmDo0qSNiiyBv3DijhQhDxTrl5IOPTgmmXwHpoNyBoW6lS7bx7y3DbEJB8a9ZQi98Sp0liJcHGdw8eFy6crwIDlM8iBRVRrJJWKafS6A1Yl0iJIQKyYwSgvWGgUjWJzhRJUc5NBcINAYUYoHjcmsq4gbo3Yve/wKC9NnKSSMJO/bwAAIABJREFUhqR21M/SYo3B/eMklQLcMOK232u0G5CXt+2xvBM4qaqvA4jIbwLvBV68br+fAX4O+NF2TtqOYflvMaXlvx9O/lXAd7W35vsDte4q81ML/P/kvXmsZVl25vXb+8zn3PnNQ7wXc0RGZmRNWWWX7bbbljFNgzBISFhNt9Q0SNCAEAIkJJCQgH8QiKElpm663eqWaISE21BtGU/YbpfbzqosZ2XlGEO+iHjze3ee75n23vxxbkRGREZmRuVUGcUnhd6LO+xz3r33rLv2Wt/6Ptd/L4CoXCEt6wO9ej8UVgN0v3DoM1Gh7pa/gx/G+FEE+iba2Cyc/Reor75FOnsdS1q4TsLhjkbaYEgxSpClNmHJMOjYrJ3NkLLozhzdtUlTjesKtFYsuUXH5oeF0fNW81OW+UvlnKUNRTwOEBbkucPa1pColtNvuoz6mlINjLXJqO9QWSoRVceFNKWZFQVZEYH75WLyGUCdFJKbwS8Agm6rQWv3NRAhixt16ssSKS2EtVasYR6NoJWFgM1LDoe3m4WvkEiIai5bLzRADoqtlfP0xLovGsSHF28XhRAPb1v+1lxq5D42KLzD7uMA+IlH1hfiK8AZY8xvCCE+ncBijHllvrgxxvyrT7PojxtWthdpH3ZI4xTXd1G5YtgZsXV182NZZwprC6M7hdeynhR+yyYGe4s08+idSCy+j2XvE0bgBR5aK0ZDj3YroVzVVDyDEQLb1qRK4IXmQUYSTyEo5ZTcIjC47kcT4/QH0MLzuUyCtIoPsL5vsfyk7McBB8XyukWSRCgVsOrN8JyidnLly2UOdhK6bQshRyxuNNg4P0TYyyTTAVr4uE4NS6jHqngBMMPoCXs3exzfGROEZxCmxc7377G4tcH5LxUWqwYJwrxPK2XjQp3F1ZQ4XsR2bcKKg8CAnr8wzyjcj94KtY0xL33I/U96Jx+Eqvlu5b8D/uoPc15P0xX6JvB3KPT8toQQXwL+DWPMv/XDHOhZRlSNuPL1i+y+fUC/OcR2LLaubrJ67uPtCIWMwP0aJj+C7BUQPlg1MuVztNPB87pE5RTImYx9nGREPB4z6FqgBXffDqg0bJY2EqKSRiCoLxUckywrgkQQvTfiKOYMXG0Kr7LH8Xht5eHrTFqP0f3ve5Q94eNY5HMVpFwlKF8CMStU3/QMxAg3Cjn/YpltBUJqJEdkusb+rSHTYR9FjSBQLG9OiOozkA6YSdEBEhGz8ZTTe03qK9V5vWUFvwqtoz5RY4LnZ3iBSxBdgHwHI+6TbBJwXsCLBF44ncs0NAstG7EI7tLHeh+/CEhzxWHnE22FDoCH07VNCtXG+ygDL1CYFgKsAt8SQvzzH1bAfZoE978H/mngWwDGmB8IIX72hzv3Zx/VxQrX/9xz5FmOZVuPmI39MDAmo2DIekjnAtpMwOyD0UwGMUZnBGEfrSyEMDh2zKTXZzYzBL6isp1jWRbv/FlA88DlwgszVjYT/Oi9AOF67wWR+5mIZb0nyvQwLGs+4/MQxFx3RcqHgso8mNzPXh4IPz2kapllYDtjBPugAnC+CnLvIfEWG2QNy5wWWxbh0TkeoZIR1YUylnVInEV0jkbYXhcvpJCvlEuAJp7eT5VyjOqAHpPlLoe3+oy7I2rLVYwxLKzX2b72JSQdwMy3PzWMtQrTX4Pk7SKYiyqFK2MHY9JnjnULRbt54xMUb4FXgEtCiHMU9dNfAf7S/TuNMQPggeGVEOIPKaRiP3FXCGPM/mOdj09HIPUZgxDiiX41TwNjNEbdm3cxKAqQ1oW53IEG1SSNS9iOQpgRRhQTyXmuaR56BKUpYZhhjKFc12xf1QzaDmvbKdXGo1mHkBTNkzm95b6GimW/R8+37bn+ypPeSfHeP62L55s5EUta71H7n9RFyjML25EIcwd0ubBDlQuFfxCAahcZjGqRASodEFV8LEeBsHERJFnEZAxe5fy8gJCBfQ3bdTE6gWyXwrDMo3nvgHjc5+wLP0NloRgebB/2KDfKLJ+58OjJyUWM+yJY2/M/JpxboYww6vh9pLxnAWmuOPoExVtjTD5vyvw2Rbv5V40xbwkh/nPge8aYb32cdZ/KFF4I8VOAmevU/rvAOx/nYA9DCHEPGFEEqfzxfaAoItnfAP4iBV/9rxpjXv2kx/1RwahdyPcKjREpC6X3+LdB1MHaAt3Bd+8waFo4tsbzSmCN0EqQZw5pajMdZ4QlhbSgtgiN5RzbNe+7wIUotGj1fHKZ+bwPFEHBsp4QFB7b3khZBBNjChVJYwrxpwf3fcDfqbUhzxwcxyqU4NxVcF4qSGxCFtHOmpErn6PdMntvd/DDExzXYmnTw/XHxMlzuLN0rkS1hbALslyppgmjEaPelFJjAaOhuZdSXSgRRV3uf7FG1ZDmfpvlM487SxYGbeKxQq0xLmQ7aGThbijub7X+/wFjzG/ymBSKMeY//YDH/vmnWfNpAsu/SXGBb1Dsx34H+LefZvGnwM9/iI7tP0MhoH2Jokr9P/NYtfqzxuP+ysZkxVSsKQYHhbX0RO6DMTGYlMI9zy00VNThXBRaFhyK9AbkrxXiy85P0B38OfZvfId+64R4usTCcs7G+RGB32dhHQbtHKUl44EhLBcDgJV6/kDE+kl4eF7o4Y7QB2Yp989fv7cdwhSZzuPjAPmTspXcRiuD1jHGgOUarPv+y1ajaAmjwHmJ453XaB/GxcS1XcUIxdFdwcYFSRonhGUf6ICpUGz7QUrJpS/77N1o0G/N0EbjhS7bz68ixRij84IsV7wJhT2KmSv7izJggZCPFHaNnkH2BhCCdDEqLxwB7KtfGH/mD4NrW2w0njHm7Zw881eMMf/K53Q+D+OXgb8/l6R8WQhRE0KsGWOOP+sD95oDDm4dEY9jgrLP5uV1qosBJnsd9PQBD8OoXXC+9IADYYwqZCPV8Xw/ojHWGZAb84poNi/YvjE3D1NgTpiOfos3/rHEcqvUl0s4dheTn9JvJgRnJWFZ0T5wGSYxQdnCDDW1hqJUNTxNqedxdu0HdYAeQIDS8wBz/6bHv8AdHjFO1AowBp1LtDDEU4OeZNjhPn7+h9jBlwplOKvDqJey81aOLWLi2YRxp48fgRP4tI996uslSvWw2D7pHka15wLY4PohF1+ckSUZIFlaa9A96ePV+pC/AViMuwHnXtzApN+Zv+4ChItwngfrDOR3MXPlf/KdeevsQlFUp5jlMmLhwTG/yEgzxWHrGRtCNMYoIcQvU7SbPm0Y4HeEEAb4m4/11uHJ/fUN4JHA8mn7CvWaA259b4eoGlBdqpDMUm6+ssO1r7tEldkDNidQ2GzmdxDui8X/8/1iulYuzOdVdOGnbAVFW1ndLlil+R7FDtBCscYb/+SI/Ztjzj0/IU/bpOOESt1CKUk8SQiimKsvCeJJhs4VXmgKKcinKfeI9zdx3j9VVNygH8pUhIAsLwrBT2LlOk7hTqSVg9aacR8O7/iMB0ULavNCRnnBYnia4ntvUq4d4VbaWM4K/cM/YTZWbJ3vkmcW8TRi3JvhiylRxbB59hTBRsFnwQJ1jGZW1GeyW6APcJwzICQbZ7tMO/fotc8jLIHJmywst2iUvwfqebC351ljXBi7O18tPvXqAKPjIotynn8QVIBiZkg3gfcHltl4xsm9JuPelKgWsnp2mbD8+dqqPgzXtthYeMYyljn+iRDif6DwW57cv/FTqHf8tDHmSAixDPyuEOKGMeaPHrr/Q/vrD53Hp+ordPjuMWEleKCD6gUuRhs6hzcIS9sMu0PG/QmO51BbquC4fbQaFMEi/eNCZd5oDBZZ0iaLEyx3iufNCSBmArhzUkibXusM/U7C8vqQSqWL46QkwHTq4LgCpQwqV+S5pFSdPSimPrUsgnlv2yIcEPkH1Efur2neq6H4wZODilYwy2DQ98lil3hms38LKgsxlXpGrizu3YooVTxWz07RuaLXyXHGt6mtRUxHEI8n7L9rU1/OCMMpUi6wv5Nx9WsWwmqgjcWo3SMeD/G9MeXlTaQoBLohADUAGeE6Cc99o8p49hxZfIjvO4TRKsLsFUEjv42xrxZ+y6aLYIKwt9Fyg+mwi4ozgnKE8/Dr+QGEn8lwyjt/ehNpW3ihx6A1pHPY5do3LxNVfzQDjGmuOPpk7ebPBE8TWH5q/vNhU3YD/MInObAx5mj+symE+HUKavHDgeWj+uufCabDGbWlR6UJ3cBh3M/Yu7nHpK9wPAeVK9r7p5y9ViEov05RUA/B2JC8TL+T0j51EKSUSkdY3hoLG19BiBqIQaFML0q0j/pYcsJ0PCWeeUzGDirNico9bMehVEmRcoYU6oFO7ePbEqPmXJWPgMmeYsJeFOvfP0Q8KyxTe20bx9VUahojBMnMQmUhruewtyOwnRTXdYmnNpZtCErQOrDYutxH6xIGD2NiOkf7tE9CGitTdm+FtI4kfhAgvTXc0FBetMnUArdfbTMZ7SDlGK1DSgtjLn7JwRYBSVIGEeOXroA4QJouFT+Zj1UvzG1RBAgfo6bEk1NyVcYPUlzHkMwSdl67x2QwRasUyducfeEci+vlIss0M4Tzfn+8o3ePsT3nQYbiuDazcczB7WOuvHTxo9+AzwrPomyCMebnP+2DCiEiCvX/0fz3X+LRwAUFb+bfmc8u/AQw+DzqK5V6RDxJHpkNSqYpebqINb5LVFtl2JnSb/bQqsewG/HCz1xFqxLkPkHUZtzZZ+f1Ew5uB3hRn/qSz5mLfQanPrXlGqi5Ti2QJCUcx1BbrGP0DEsMMSLF6IT1zRQ/KtIFdT/rmH+RGv2e5cYjwtifIlRezBplKThuzmRkM+pb1Jf0nIrr4DoaYQxZIhgPQWUSP8wxSiGdjOl4hsoFmZpSWVikddCjXF9G5zO2KdFvJ8SjNrYe8KWfeR7LHbN/s8OgOaC2XsN2NkC6DDoJd98YEfgHZFkJyxqjRJXNCyGunT564sIHIcgSzelui+kkQ+kKUk6prFxh2G0RTxOqSxWMuYKK32XntVv4wQZRxQH7bNGtewyD9ohS/dHMxI88hp3xZ/DqPx2eyeLtfQgh/lngeR7VY3k8EPwwWAF+fd7Ss4F/YIz5rce8hX6TotX8LkW7+XMZJ9i4vM47L9/CGIPrOySzlHSW4frLYEmGrVu09tqkacqoX2bcO+LkXp/z168gRB+TvEz/9ATL1mxf6+FYklHfsHdDsHXtHrXGBuAXNRc9wvcsSrUU16/i2D2CYEo81bihxPbe24M8PKsjKDgoSVxozH4WnVGjiy9C2y7azLYLnp+jlGA6tpF2iGVBmmiqNZgMXSzbkOcB42GKEAmOm5FnLkK6uG7C3bfa5IlFbU3Sb3kk0yEq6RGGM/rDGc3dW2hzlZd/J8X1yjRbdVa3oLGc4ocub3+nyTd+McQLNeAzGAiO7rQ5e6VUvComn1uiJGBdo7O3i5QjSvUFjBHEyQvsvH6AVob1C0X9RAgby7+KHbTo96qUls4hxJOtRMJySJbkj3gRZUn2vgHVzxNZrjhqP4NbISHE/0IxhPjzwN+mMBP77ic56HyS8ktPuP1hbyHDp9fWfmpE1ZDnfvIyJ3dPaR10SaYJSMHxnSGOZ9M7rRJPBK4XkCuHYb9PpT5iPOyysTWmueeze9Pj678wRGWGeFbG8WymoxnD9gGeN0KpBk7pCq5fI0nepNtKkWKISh0aKx5LazO00uQxWGGRpdxnwWpdZC+awhtIm8KF8IcRyH5a3A8uShfbrSS2UAqGfRcvgFLFYDkWy1sJvW6Oym0ay4rpSHLrBxUW1xP67YiFNZdBN2U2cfACm1JpSJqs0Ts9obRok0wW2Lzgo5XLa79/E9c7S6lxCWmXOd3vYjtD8txFSokS29jsoE1AWMrot0tM028SVqZgBoUpu32WNAkZjrpElVXS9CJa1zC4OG6P9lGXhwuzhUtBhMqjDwwqAGsXVrj5yrtIKXA8hyzNGfenXH7pwgc+53PBF9DL76lqLMaYF4UQrxtj/jMhxH/Dj6Fi/7g/Yf/GIf32EC9wcX2XZJrwzivvMumOUXnOeDAjzxRbVzdQuWHQ7iDtJfxSn2HzXRZXVui2wLZzmocWfhjMmbSCycgiHIQsbQVos8nhnTr95j2CsATCobqYE48nzMY5UuakSRE80rQYItSiKCGYeYvHceYtXuYOhZ9yUBFzgtxsBp4Hs4kgzQo92zS1CQKN546xHIlShkvPC2Yzh1HXUF/U/OQvTRl06yRZyP7tgN3bLqsX1xj2A5LbLtlsj4M7HgvrK0g5xS85tI5SLFsR1epMxhaVhiAo12geWth2m6WNHEtMmSZfI1crgMUsnqJ1iJARxvn6fIs5BXLi2TbSu0BR/yrghS62LUnj7JFp9TTOiKoRrYM2KleU66X3FWTry1UuffUcBzePmQ5nOL7Dxa+co7FS+3Rf/B8Cjm2xsfhsboVm859TIcQ6hRPiuc/ulD5/TEczvv//vk6/OSTPFGmc0txvk2eKSX9MqVaislgGIXn7T24yHU5ZWKuDkNiO5GR/nfriDs17XYwSHO01CMo9BIYzl6YMex63f+DTOxWc7I9YO9/E8U6Z9fYohSG1xpSDWzFBmCEsSZYaytWcNJ4biM2Ls44zlzCQwFweQTJvD9ufbsaSJ0VtJU8LM7OTA5/WoYPKBJZj4TsOk7GFlDZRWdFYBce5z7wr0TxWZOkCK4sRw/aYeDrCts4QrW1ggOM7dxkPfBorMStnK/iB5t5Bh7AM9eq7WHbAqGtQqpih+vLPXsaStzFG4NgtwJAka0gJYXSESU+KF8HMRwNKL2D7R8zGGVFlhGOdIJgxEzbXf+Y6J7sj4kmMtCRZkhGUA+6+uV9MPAPDzgg3dDlzeZ3FjYUHMqQLaw0aq3VUrrBs60fO0M0yxdGzxmOZ4zeEEDXgvwZepciO/9fP9Kw+Z+zfPORw55RyLaJc9hl2De3jLgc3T7j80nlm45jdW4dMemNsTzLqjdm4vM7CapV7bx7QPbGQ1hmkYxj3aoz6O4x6M6LKmN1bhqOdHEdOMWbG7e9HvP1Ki63Lfda3R1SrRyRJwIUXElQ6xfUN1YUE14PmgYUxglJVIYRAa02lUdQ9YO7dI8Cx57NAn0JgyXPotQWjrks2E6SZZjqWdI49GssZYQn8KGU0gJUth/qKi0QDCRADEWBTXTAMB0NUMsSLQtIsJBCn1Jc2SLMyi+sbjPo9Ni5IvADQI8KyJJ44BJUS5YVbNO2AXlOxsFxlYUVjxCq90wm2I7DELmk65dwL15Bi7wF3CObM5/wdVs9e5O7rr+LKeySmhNEWK2dcls+csHjmBfrNmDxVlBol7ry+S1j2cVybg9vH9NtD0lmCAE5325y7foblM8UUtBAC2/li6NA7tsX6s8hjMcb8F/Nff00I8RuAP594/LHByd0mjmPjhUVRTuUaowzGGIbtIZ3DHo7XY31zQHXJI41tdNqmc2RYvbjC4HRIlpXYu3mEJWKe+0aNfHaK600YD238ksCxM5IZRdGTKcd3DRiblc0Yz+/j+zlBlKNygevCdFRsP4wWJLGL4ypKVV3UWHSxTckzsObKAE8rxPRRaJ9aDNsO1QVF6giyrsPBuz6XvzQjKBmELTFYhJWEkz3Dwsp9QW8LqFLU92M8t8bm1asMmhOMPmXrSp3uacikd4I2CaNBhYtfWUBlrzLNYozxUNqmur6CoUP7YMx0VMOSEWubOxy/6+NXX+D89W0m/QnSWqRcV7hhAsZ7JHPIEsnxuzt0WjlBeMJs4rO8vcTCWgM/8jC6j+t2WNlex+T7TLrvkE6aRKXzTAY+o06bxZUJ8aiLaw2oL19k9+19Gqv1L0xAuY8sVxw/oxnLAxhjEoqvph8r2K6NVu/x3KUlsaTEcS2O7zRprGRsXZjROrbotSTbVxqsnss53rPYfvEClmXRPuiQTqG20KJzOEUKjyASQEqeapKpYDoWWLJP69SnsZJjWynNI0HgC6qLhtkUHEdiObB72y+sdlzNdCAJy4KwnBfTyXmx7bGd92ohAJhPth3Kc8gTSVTWZLHAdiAqK5SSZKkkQJAnkvvGaNOJRbFTntsgUqLg+kcgbHzrBG89Y9yfsL7Rx4/W6bcyMr3J5a9alGoTHGeNaeceWo05+0KdMxdiukctmnvLZKkiajSQrsYWQw5u7LKw3qBcLzHqjhm2h5QaDfzoveKlMYa9G4cIo6guRniuTZYGDFp96itVht0xGE1YamK5LTAZwi4j6EK+SzySlMonCOGjdIgRAb5/hOeWmY0vU65/8SQsP0JB7keCL1b4/RFh+9omu2/tMx3N5kxbjRO4uIFLMksolVq0DiR5Bk4ASWphdMD55yWlRpmF9QZSwPHOPU4PXLrHI65cbxMEKc0Dh27TotbIiSqCLBEIK0WgWFyLufbVCWmsERLCUOMEitm4aClbjgENq5sJXqSxHLDnA8KquLYLDWnrId2Vh/4uM7//EZLLQypwjv2on5DRgDYIu8iUMBppQaWe02vZRFUwwsOSM4Y9m3LdI9FVPDmiGBwSRZ3D2oD8DqgRcbJOv+PhhyH1hZDq0hbt1gq1RgsjzjDtu9SXb+OHUGpoHJlRqsPWhXtMJgsMek1uv5bj2hPGkwmv/eE7bF5awfMLlbi9nSEXnh9QXY4QwiKexGRxn6haQ+k+rn0Xx/ZIxzNuvNzEC1exrCGOM2blrEtU3SYoNQgrEbOpwnV3mGqNyisolVBbjNDaxnVO5w6PXyy4tsX6M1q8/bHH6tllrnzjIkc7p8zGMbZjYYymXItwPQu/3KTXhMZKlVItYjqYcLwruPK1iKAccObKOrlK6B5LWvszjGqTpyl5rnFcg2Nrui2Haj0jTQW2LVCZZtAVuH5OVHmvXgJF1lBfyZn0LbIcgrLGsQtqvVJge4CBNClE1gQ8YF8awwO3wsLutFg7jWE6EsX9liFPCw5MUHqoZiMBJDoHg0Fp0MqltqTIEjjZLUSWwEK6FvXljHtvO6xsOkTVEEcGYFXBDEjjKVlaZTRq4XoSwxog0DqmXDdMBoZLL60yG8KweZbJ6IA87xNWDK6fYtsDstRDqj4r62VGfZvZ0ZA7rRkXrtcJyzZJepXIiti7nXKt1sGybYyaAhbahDj2KWm2jSWP6DcnNNZygmoJ126S55L2wRAvCLDtJheun+P2D4aks5Rh3yCdKdtXlwonyuGMsBwQlL54085p/uwWbxFCvAicffjxc6P4HwvYjs2LP3uN9fOrnO62uP39O1SXKnSP+/RbYypVm1LZMOyOUbnGL/vUlhyWty+yfGEVy7FYP7fB638gWdwIiQezomagMhZWM8JKzuk9l/EIolKO1ob6Ys7iWkr70GZxXWEMeEGhW+uFmorQZIkgmQmMhiSReL5BKYPIi0KtmVPv0+TBMDVZOg8QBnJVGAUaBe1jCz8EbQTJDA7vWNgeXHguwXKgVCnqNOV6zsEdjyDUxFPBdGKoLuaMeha9pk2eQVRRNBYVUcXDsTvoHIZtqC3VEWqN/dtHnO5ZSKtKv5ezvJmxtJmhVA1kSK+VsPODfb73e4eEwW1e/OYhQeQhgElvSrmu8cKI1r7GcXws+4TW4SLN4yVUNub4nsvGlecwJsCyIUkWmcVblBsCr2yYxvcIyrfRugxIpoMUrVoEkYVtHZOpdYyxEeYWySzHLof4/gnP/+Qlpr0R56679FoCrRSj3phyPWTt/H13gS8WXNtifekZzFiEEL8KvAi8RUGtgOL78ccmsAA4rsPa+ZViwNC1qS9V0c+fwds55WRvynNfnWKPDEIaVs74YDJaxz6jwV2cwOXSV88RhMu8/I9+j0qtR+ObMWkikAhUCqvbMVuXY+KpIJ5YLG+mCAm7Oz7vvCpQuUW1kXP+2oTqgsH1ICopag2NF4KZ14DSpGDcun5R4FV50ZL2/YIda0zx5uQ5DLqCxqLBULxxg46k37OwbWge+Yz6FoO2y4XnC0nMch2qDY3txkVr2Ui2r8wYdSVUobagmU0kWSbotRykk7O+rXFdyWjkkcUz+u0upwdL1BY8hG3AW+fGD3KMqLK40uF4P+Lb37qFUPeIqjO2zu7TO9E4ZxS2LXGjiHjsUmqUcP1iJMANapQbdaJBnXhc485bGasXvYe4OwbL8REywvbg3As5rZ0foI2LkIJh2ybnCkosYZtdtI6wrUMst4MkhbwGIkQ6NUqLX6a0OGBlOyVNPKRU2E4C9nmE+HjqgZ8l0izn6KT/oz6N9+FpMpafNMZc+8zP5AuANE7pnQ6oLFZoHXSo1CLU1gKn+21uv+mwspmwsGgxGRqOez52eIzr3WXQSvjtv/tbHN8doDJFVIbltRg/ytBaYNk5fgBZLhDScOGFBCEUO+/4ZFOoLuUkE8ON1wLu3PD56p8bUl9OkTZ4PgRhsZW5z7zVqshK4olFpaEKBm5eZCu2CwiIJwIpBMOeoboIiyuKG686OC6MB0WwC4JisHE6tLBdRZoIFtcUQWTYvJCSJcV6nRMfrRTjoU2pkjPoeliWQKcZk4HAdhSO45Gq5zjZTwnrCwynEemsx2zcJY/hrT8ZcPWlCt/9/SHNwxkrG2tceO5V/MAwGcLpfkKpVmJtuw3U8cI1vLLPaDDElgkQsnl5jaOdPhiIJwlhOWAymBJVQsJK+OB9XFhbIQqfY9QdoXKbtfMrHNw6BjMhV3Xy2Q4Huxm90wbnng/YPB9TqvbB+iWEc5FCN+cA128Xc0fyIsJ6VI1uNokZtIdgDJWFyo9UOuGLqHX3NIHlT4UQ1x53Rvtxg1KK3umA47unIKB70ieZpgzbI6aDMVLkgGQ6bFNbOmBpNcDKdxiPGqQTj9beCCkCVs9bDLsW777tc+6qplpTNNYUWgmSWLCwpJDSIKQALYlqhrCkmfRtLGlYWksZDy2kdPF8zWwM0tFoVWQjWr83zBqWVXGRjYqCbFQpire5KoJSY1xIAAAgAElEQVSP7c7p6hJSDeOBjeNppkObcl2Tp5LaYsZ0LFlYA5WLgnQ3V/t3fUCA5xtEaOh3wFtRrG/OmExt/JJFltmMB5JyPcMONHEyIx3FjPsJR7sNhq0uWZaRxgGZaNBvZ2ycX8XoNlnqMR7YpIlkFlv4kceoFxNWUhx3kbWzLsl0QBAukLNEkoVsXAoY9ce8+/07GASNtRpf+8UvvY+o5pWv4fo/oFCNc9i6EnJwO2bQK9HdO0LIGsvnFkkSh3deHXH1Gxcpl8K5apyHcC4AT6bqtw473H1jFyllkSHqQ7aubrB67vMXhnJti/WPY/P7GeNpAsvfowguJxStZkExyvPiZ3pmnyPSJOPGd2/zxh+9zRvffodhe8SgO2I6nGI0lMpjokgT+Jr6UsrmhYRStYPthCC6vPXdKuvbhvFgwqQ/o1yLmQ5t9m/7iCsxpYZhcUUzm87rHxImfUmeS2xHM+jY2K5h7VyK62omQwuVSaKqolRRTAaGIDJkKUzHFu0jh35XcOl6QhhpMgGuDXFcrO/Yhc/ysGdRrhcq2MaA48OwZ2PbhlJZ021BnlpEZYXrgkFhW4VUglaAU2y9SuWc8VhSruZEFYUUhuRQEkYC25VMRhJp11hd9AnKiuZNB5WPSSddpF0lGRYF2YObMcOuJiiF+F6KH0xJYoEf5EhLo1RIv78ItqSBYv1cTGsPhv0ZWsc0VhycsMHd1/dYPbdCEBVzPTuv3eXaT14mKL2XNQhZBedrGHUCZkpp8QUuVuu8/kevUV8/R7Uhi6K1yRDWEod3HK6u5E/6eLzvs3LvzX3K9RLWXARYK83ezUOqy9UH5/R5IcsVxyfPZvH2V4G/ArzBezWWHysc3zll9+19Tu61qC6UOdo5ZTKYkSUpWuVsXZyxtpWxtDGlumAhRIJOM7pdm/FAMezDyW5Aud6jsQwv/UKfai1BK83CakalbuZBReAHBg1YdUVULrotycwCAfWllHhiFa1gx2BbBUluNnGxnIRJ3+N032M6ljgO7HmGMxdz3ECR5wYpBElqyGWRdQjLPJiClgI2z884uOuTzCCIDN4YpmNBfSUnjkXRwp2rxzku9NqS2VjOWb2GUg3SRBKWNJWGxnIESmkq1RSciCwbE4QBg27E6Z4hnU3wgj5Cugz7S+Spy6A7ot8acf2nbVpHLkFZksUxi5uaqDxAWmXe/P43GceX2L50yNbzL3Bwx0FKB6OPOL7bZ/XsGpVGGS90EUIwGUw52jnlwpfOPvK+Chkh5HtZh2sDIqS+soGUwXwPIQkcSb95gpDvl0p4HJPBFKPNg6ACBe9JIJj0J597YHGe4Yxl7+NaADwraO23Obx9zLA95Ohuk8lgQpak5JkmCGH7uRTLUljCgE6IyhlhlDOLx8xmPkvLCQvLIzwvZ+VMytb5GD9SWLJgxvabEq+k8cNCo1aKYiJ5ZTPl3s0AxzEc7zkYBX6Ys3YuLaQRMokbGDxXkaSS0z2bhbWcUiKo1nNUDoe7Dmev5BglUMbgBvfb0gIpNPFEkOcG15bUFhXSirnx/ZBRz5DGsHImJyrPFfiNoHVsULlFMhUM+hb15ZywbEhnkqiUk8QW04GFX4IstdDGo7aomY1PuffGlNahi2P10LlkNIwY9Qprj2TSY3ElZWkjYDLyGXenvHm4xIVrbc5elSytuUjbMBycRYhruH7KO9/z2bhylktf9cgSxXQwYO/mLQatEoP2CD/y2Ly8TlDyGbSGT/VeR5US6Wwd3zukmLSSJNMZYXUTROWjno6U4sE80cMwFAFmNonpHHaJJzGVxQqN1dpnytbNsmc3Y7khhPgHwD/iIdbtj1O72RjD6V6b6XgGWuP4DrNxjM40aWbw3IxBV7J1Mae2kGNJw+5tn37HYtSzWV7PqC4mWJbG9WA8sAhCheND61BwvBvguIrl9YzF9WJQT+VQqSvOPz/lZM+h27KIE4G0JSd7DosrCj802I4qZpBOi8KrEALH00RljcoEg77BkuBFpmg7C9CpRBtIE8Gw69BYzfA8RZ4WTNrr3xhzsOOyfS1HZYLZGMo1ENKQZxadU4s8lSglGA/sOddGIaTBcnJOD0NyBYaASl0w6jsc3Yvo99cxucEPOqgsw2SKeGpz/loPyxJYXg3HM6xvT9FGsPODiOaxTWNlRDyd4YUOcTxByLe4+bJmFke0j+9QrkeoPOZ0t08266JMQmO1TjpL2b9xyMblNfynzBQ2Lq3xznfGSPsirjcljROms0Uuf/36Uw0UlmoRtms/sNuFQpNFSokB3vrjdxBSYrs23dMBzb0WV75+8WP7UT2reJrAElAElF966LaP3W4WQpwB/j6FVaOmMKn+G4895s8D/zdwd37TP/yEwlIfilI9wnFspsMZ8SRm2p+SxcV+W8UwHkhWt1JcT7O4kfLumyGjvoXvayYSLEdRaeTkWXFhp4mk33EIZjnxzMb1NEGp4KIc7Eiqizn9tkW1URDoFpYzVCaJZ0UBtd92WFjJERZMhjaVuiIsa+KZJp2BEZI0FViWwfWLdaWtiWeCZCpRmSDL4fieR20xx6iCKBfPLNpHFkoZGksKx1bkiY3rF+3peCLxouL8JwOLc8+leL4mzwStrovrKhzfsLqZYJBMRjFJ7NA6Mrz9/RLVhQxUn8lgRFRyybMOtgWNpZjuSYQXTggCi3E/pbqUsro1Yu2cxglsdl4X1FcUSpcxYohKYxqrLraVcu+tY/ZuDHE8i8ZKhe7uIUZrljYX6beGdI56fOUXrj/Ve11ZKHPl6xc4uHXEsAt+qcGVb6xTX64WouC9CXmW40f+Ezs9lm1x6WsXuP3qHaajGSCwbYuLXznL/s0jvMh/IAQVlHyG7RHtgy5r5z+bwq7jWKx9wq2QEOIvUFj8WMDfNsb8l4/d/+8D/zoFO7IF/DVjzO6Hrfk0Q4iftnJbDvwHxphXhRBl4M+EEL/7hK7Tt40x/9ynfOwnYvnMIo31Om/+8Q0mwylp+l4RzwtSXN8gMLSPbaoLDqWKwrY1zQMXz9MYDZOBIE0k69sJ9ZUcWxomU4tyJQctGHQkQbloAffbNraj0cowHkI8LozMg0gTVhRL6wIhBaMeVOoGZQQIQ6mq2D31KFU0p3sOS+sZlVqOEILp2ELnkGYCCYz6FsIpxFuSTNA6sWkeOMRTi5WNFGEZkqQoHgspkFIQlovzMUrieGC0YTKWaCXpnjjMZh5rWwm9lkWlZug3Ba0TjzvvBORZijQZUvbZujjFdjyktAi2NZV6RhprqosKafWZjhKqtRFGeWxsJ2SJZGUzonm0znQyYdLPWT6TMO6ckk0dJn2PSkMST0DILZxA0Nxrk8Y57YMOeZbz7p/tcPb6FrXl6kdmB7WlKrWl6iO+Ucks4daf3WE2mhWT5MawfGaR7Wub77PTLdUiXvy5a0yHM4wxRJUQpTTxOKb6mF6yX/LoNwefWWDJMsXx8cfnscwtfv5H4J+i0Jl+RQjxrceux+8DLxljpkKIvw78V8C//GHrPg1B7u/yZHX8v/ZDnP/DzztmbuEx17x9h8LW40fSzk7jtBhoaw3xIw8ncGjtd1BzF9nlM5rTPY8sMSyfyRj2JKVyccF2mxZKCbLMpbGUs3EuprZUtI+FgOkURn2bPC9av1IWDFmtNdUFxfGuQ1RWOL6mZBXbEt8Fy83xfDFnvkpcTxCEGoHGkobJWJBMbfyewo8UjqcIgmLaeTx0SBOLw7s+476F2o6JqjmHdwqCixAGPzKUKjnCkqSxmbegLURYmMlrDZalmU1seh2bXtOmtpATRpogUhzcc/nHv17BCIvJOGDSF/iRprESU1uYMehquqeK2lJOY9FgO8Uw5nSUU1vKcByXLJWkcYTlpOh8yvo5i85pD2FWaSwnBOGEwx0L29PUFxLGvZCT/RDbO0I6y9iuTxKnSKuYafre736XP/g/vs31n73OxS9vs/XcJl7w4UzZhwPG3juHZElGbf7tb4yhuduislAutHceg2VZ7xtIFFKglUY+pF+RZ+qpt2kfB45tsb76iYSmvgG8O1d1ZK4x/cs8dD0aY/7goce/DPzlj1r0qfRYHvrdB/5FPiW1fCHEWeArwHeecPc3hRA/mB/rPzTGvPUBa3xsX6HZJOadl28xGydoDZ2THlmckT80mVeqKnpti9ODkHs3NOtnHZY3MxrLCs/XxFOLO2/7XLw+ISwZXFej8uKCPdrzmAwkRgskhrCS43iG89eKbpDvGywpyDKBH2hyB9LUwrM0w56F4xXrj3qC+qIi13DhhZjaQs54BKUqBAFIu6jZZFOwHYO0FK6T4UeCJBHs3vLpN10cz2C7mp03LRprFtsXEowRDwYek5kkS4ptU56D62scVyMlTIYWUVUx6jm89icRt98MqS8ZmkeS+lIKRuF4fRxHcXjPBwFLaxmOm5OlDvVlyFLBZJCytJYSVS0aK10mA4kfQJ5Puf4TktMjF5VNOLpTtMVVHtA8ClAqJpspjm5NkG4fpWDlzDpnr69gshs0FjLqDWjtvs3Ceo14mvL8T11B3Pd+xnngfpilGc29DoP2ANd3WdpapN8cFGJeFJlaniuCckDroPPEwPI4LNtieWuR4ztNqktlpJTkWU4ySd7Xrfo0UWQsvQ97yKIQ4mED97/1mIfXk/y7Psxx9F8D/p+POq+n2Qr92sP/F0L878DvfdTzPgpCiBLwa8C/Z4x5vKT/KrBtjBkLIf4i8H9RWK0+6fw+tq/Q0bsn91fh5N4pwgjcwEWPNWreWe+e2nhBDgLyTLB/x6PbcVheTTk+sAl8gespglDTaUr8SJPG0DrysTHo3MK2FcO+xcE9l2/+0ogggJM9G8cVJLHEDxW2Y5iNbYY9B9uRJLGF5SomA4fOicOgk1FfyUmtHCME8diiUs9BFqzczomFus/nN4W3c5hmTEaSt74T0FhSuH5OqaJwXMPJXZ8wNCxvpkxHFhhBryUQQiAEJDOJZaesbGhG/eLitx3NsG8zbjl4fs6gK7Fliso0rm9IJzZJZFjdyKgs5KSZIJlIcq0xGOrLKUGQUWlomscV0jQjDBJA4fkKRMzZS3vs3iqze1Nge5JKdYLl9ilXUqxtj4O7EYMjQW2lxOLalLj/BvvvpgSlAMe18YMDbKvDbLzApHOLqNJ5IAhsxDb7O/Dtf/gyBzePsG2LcqNMuRHR2GhQXijTPerSOuiiVTG/tXll/ak/TxuX1jAGmrstQGDZkgtfOfeZSy2ID9e8bT/ui/74059w25Otp4T4y8BLwM991Dl9nD7YJeATWQ6KYuji14D/7UndpYcDjTHmN4UQ/5MQYvFDfJ4/FnqnfbSGb/+ff0rn6D5DNHuErdM5cbjwfPbASL1SNtSqil7Xpr6g2byQEoSGxopiMrToty0GbYeDux6VmiIoZWQppLFFUFLUF3NsH/zIMO7bqEySxLCwAkiB4xWt3b2bHuefj6nUMixb0zl20UoQlgVKSZY20oLir2AyFNi2JgwN7WOLcd+m15EEIVQbOX5gwDKc7nmovDh2+9SmumSxdcmgMo3tG5QSjHoOqTGEJUPrxGHSs+kcuTi+IarkjIc2565PiA48SmWN1obOqYfRglHPIpnB5S8njEY2As3hrkO/5VJb1ERVxXgccO+mYGlTUV+URBVIpjbNQ4/JMGTtLJTqsH7BpVzRuN4Mx8vodyCNFatbQ/zQYu9WzpvihHLdJks9ZuMEjEBagsb6Kyi1QrXqcubqVYJSgDE5nf3vcPsVQ/PeAWvnyhjjEE8KQfB7r+8igfFgSlQNkVLQPR3Qb/Xng4gfHRwsy2L7uU3WL6yQpzlu4GJZT+ss9/HgOBarn2wr9FT+XUKIXwT+E+Dn5rpMH4qnqbGMeDSCnQD/0Uc970PWE8DfAd4xxvy3H/CYVeDUGGOEEN+gIBx0Pu4xPwiWbfHdb32P8WBKqR7RPGg/RgE0pLHm7g2X5c2UxlJObSmn37TZvjpjYTVjMrRJUkHnxCasam697tM5cQmjopvS3/UIyor6smbYt+i1i6wmTyS9po0XarwADnZcLClY2kzZ2/ForBbbpYM7AduXUxZXU0YDGz+ApY2MxdU5xT+HYV8gpESTE08lzcMiCNlO0SaejiVaC4ZdmzSFxlLOdGwxHlgMOqLgyniGIALPT+m1HDrHFmnmcLDjkSWC4z0PpeDclZjGao5RBmEZpDGEJcWN1wIWJxCGCmnDuGcDgtMDB6MEzX2H2VixfEahVMpsmPPCSyOkYzEbe5weVJmMNEnsIkXG2rYNKEo1RWtfMBtYTMYOtaUSZy7l3HlL0T6Ksd0QpQpfoTTOcGyP4zttLGfKq79fpXXos31tibVzdVrHM4R+B8/1kCgQGsdxyNIAN/I4vtskqoZFUVYbljcXqC5XObnX/KGyDsd1Prf2cp4pTo8+0RDiK8AlIcQ54BD4FeAvPfwAIcRXgL8J/AVjTPNpFv0oU3gBPG+M2ftYp/xk/DRzJq8Q4rX5bf8x8yxobgHyLwF/XQiRU0iU/crcDuRTheVKBu0htlfonOr04UMYigZW0Ybdu+0zHeac7CnKVcXqVkpY0jSWFc1Dm1f/qMTiesbOmwHxxCJNBGcuT3FdOD0MKM15IMIS9FoWo77F0kaGysEYQZ4JRhNJfRmW1lKSiUWvY6O05HjXJc9haT2l3Eip1gsFp3w+2VyuGjongnjsYdkQlhTDns2wa+EFCq3h9NDB9w39tv3/tffeQZJl133md55/6TPLu65q78YbzAwEwghg0G1AAEQSwwClFSmFyJVIieIiaIKURCm0YnAVKwuJWi0pUUuKIESIFKSlKFoMDEFgZjCDcWj0tK025Suz0uezd/+42TM1je7palPdXez8IjLyvcyX9518mXnz3nPP+R0MS+FntMzlxXmH6bmITkMwbIXrQbcDSWqyOO+QpgZhqKNtu01h8ZxNkuqp0vK8RXXFIop0xvb5EwZD4z1qa1lyxYRMNiVJoLlhU6womjVYPOdRGRU6TZPFcyGFIYtGbZpcuUunBbaXp9XokSlkcN0mYSC0GyZrKy6u7yKGRZqEFCsOKrUIe1rOUymwXQ83m7JyPuDh943i5cuoNOTiySpe1iEJN3CcCGWUwNAOVUNaxL0NLKvE5J4xKpNaKNvxHPycRxTG9Fp3r2iidZMjFqVULCI/Avweern53yulXhORfwg83w+O/SdoecDf7Mf6nFNKffBt7brGSVW//OmjN2z5N7f5Ra6RkKmU+gTwiVt1zquRLWRxfJdjz54g6Oh/PcNUFMsRphMT9pQu7o5Wfuu0Bdsx2XOkh2lBp2UwMRcSxwqVCMdeyNBtm7i27oxeeKaEZespxuh0wLd/tAoKTr2SwbAUjh/RqFmsLpn4nsLLpbjZhKKvaG6ktFtCvhwT9ww6LZNcIcXLKoJAR/SmiU4u9LKKMNDxKdk8xKHQbli4bkyjZrHnSI+v/UmeC6dtCpWEQiWhNNPDsYVe02Z9SRFFiigwGdsV4mVTTDPi2IseQ6MhQcdhad4lSQXDVJw/kcFxUxw3ZmXBplCKyeYj2g2T9SWP0ekA29VDv401myQy6bSgXYf8kMLzTbotg/Mn84x0QjrtHtDFy0K+nGA5ORbPJtgFIQ4VaerieIJC0droYI/ERKFFHPnkhxKyOb0U7foxjRo4uRksfwjXSYjCmFzFpr7SxPMbeIVJoqBLGqcYlkEYGTjSojSxi5Fdwxim8YYiP0Cv1WNkZmi7v4o3TBwmLN3EcjNodwO6QODmx/7epu0PXG+bW/GxfFlEHldKPXe9jd/tlMdK1NfqJLGOBREzZe99HQwzptuBw4+FuE7C4jmXVsOi0zSYnO1x9Ik2xaGYKBAunnZIQoPaukkmowjaup2h8YSZ/SFpCs0NA9OCi2dcRqZimnU9GlAKDENhO9C09egnjoTIFEwThsZjwq4QOiZrSxavfDlHkgpjMwH77utRHokBhWkpKuMR9TWbi2cNvJzikffUWVty6NStfgh6Sq6Y4NjaL7J8wWF1AcZnA6LIZXJ3QBwJlqkYn4lp1RS79gfkSilgoFSP2rrJ6nkHw1AogY2ajesleL5+j8MTMY01k8V5l/qajeungIClO0/TFDrtmGKlh+ulvPJli6BrM7Wnjes6+AWFeboO4tCoWiycSbj/yXFmjzh0WquEgYFp9zh/0iJVeUzHYOGUgeNtYNltwq5Dp1VkQmJWLpgUS2uUhguoJINSIRNzeTaqI8wcajL/6gpRlGBZsOvwMO/6yBOYtsXx504QhzG2axN0AgzTYGx25M5+Ua9FeveJ3m6lY3kf8EMiMg+0+TOU3VweLeK4FuuLNdIopTgc0dyATtvh8CNdais2nbpLp2thSMKuAwFhYHL6VY84EXbt6xH2DCQfszDv4OUSnKbB8GRMFBgkff2VPUcCbDuhvmHSbsH4roCXv5TFsFOmdke4XoppKWqrugMplHQmoG2nFIYTjr/g4LjQqJmMz4asL9rMH3d56tuajE5HWE6CKIgShesJU7sD/GyC7fRYPufg5WFmX8jF01pHxTAMmhvC2HSIaSssC8RIGBpPKY5o2Uq/kDAyGdNpWNSrFhtVE9cDy0kRU8tnNjcsKiMxYaBAFKaZosQkCgy84Yhez6K1YRCGKSImYRKi8ikLZy1MU5EmMDEXsbFqkysJ589kaX9B8d4P1ckWxpg/nuG5P8pTGGlSKCt63TZnjvksnfNwPcHNONTXm0Q9A9OrUKjkcTLC8ulV0lQxe2gEw7TZWF/hXR9+J5mhQxx5conRXWPMHR4hiVNGp1OmDj2B4+sl5aPvPMTy/CrdVo/RXcOM7hq+ZjzMncS2TSYm7lzBtKuxlY7lO7bdijuEiFCeKJMEetjuZSNUAqVygm0rOm1hY8PGsRSVSe0zMAyFZeks4NNfd4kjg07DpNMSpneH7L+vR68rnD/pEUcGfiZlcnePobGYc6dsXFfhuREz+7t6eN+vYjg5FxFHCkNMLEtH82aLCtuJWaqkfP2rJisXXFxfUR6J6bQMTryUIQk7zBxICHrCueM+paGEi2d1zAqpnl6dPe5jWSlxpGg3DbKllMpoyuRuvdTc68XUVhyK5YReR8soeBnYdaDHuW94qFhwPVhbMsnmFa8865IvpVi2or5hQmLi51NqaxZBTwfhhYFBsRIyMZtimIKXMVk4Y/R1ZRJ6HSHomrjLBqYldJrQbBiYlk11MaJeg0ZV4WVNTrzokStOsbHWZW3Jxs9Z5IfyeBmXNE5RJSgNFyhWclSX6nSSDqSKQmUIJRm8nEuzUcLNl2iurlBdWiSJtRq5kjnEfDMqNlvIsOf+2TvzhbwBoihmaeFt41juCFvpWP6RUuovbX5ARH4V7YDd8bRqTQAKlQDbVPQiXTI06EGramGgyJViLBtQiqGxmGxBEbQNxmdDyiMBzQ2D+rpJr2ORyScUyjC1t063JRQrCbkihIFiYpfoqYWkHHyoS2EoZX3JYvGsg+3qLGPPT/q1hCCTKKrLFisLjnbghibHnvcZmgrxXCC1GJ60ee6PLdYWfLxsQuCnWJbQbWr1uSg0GJkI8T2LKDLYWLOY2h0wNh1x8ZRPu2Fy31MB+XJKvWawMO/wwJMd/FyK4ygmZkPWVrqcP+5jz6TU1q2+7KVFJqujkzOFmHwpIY6g0zTJlXSW9+hMxPI5izAUDj/cpFS2Of41n2bdotcxKQyFJImB4yWsL3okaUKapFRXItqtANf3WFuo0W0oDLpcPCWEgc7f6TR6uJ7D+N5xwl5EZazA6MwIft4nSRSZvMfMoUnKYyVA8dXff5mpAxOcfbWKZVnsOjxFeWyI6nJMkl5g/yN77swX8CaxbYvxHTpiObp5p59bcMucuXcSpRSvfekEhqmYnOuycMbFzyqadZNmw2R6f0C7bmG5ekRgmAa5csryBYu998copcPuDVMoD8fEsaK6YlMejSGwyBWTfvJhSj4PmWyAm9GaLCqJOHvcJVNIqYxG+FmlExInIq3clkIUCM26QRLpTkKlKYiiW7c5NW9RqETMn/IgThmeTti1P2Z1waHXidl7f8DaosniMZvzpz3S2MDzEyw7pbaqi7dXRgPmjmh5yuqSSatmsrLg0u1YPPqeBvliSuAmHHqwQ69lcOGMT9gR8sWUKFAMjYaUx2PCroFlg5uJmdoT0m1aTO8JQRSCrknUqAopKX4+pV61KI/q+BqFHrFVRiLqtRSxoN2G2kqMn28yPgOtjQ5B12RozKaxERD0LGqLNdIowsv6eDmX0Zlh/JzbL4OiGJ4eYni6QtiNOP3yPNKfAmaLPpZtcewrFxmfixiaLLN6scr0wcnbrqVyK4jCmKUL1Tttxjdx1Y5FRH4avQzsi8ilgDWd9NGPdN3pvPT5V1iaX8bPpJi2wrQVjQ2TOIZWzWRuf8DZhsn6gk2qtOO22zIQ0YmD1pii2zbJ5vRr4yihUTUZmw7pNg1yxRjHVdiOgeNHeBn9OsOEXDHGzzrEPZNsEZpVg7kjXfJF/UPrtGFtySCJLfycXiZOEyFbTOh2oDySkB9Oqa/a5IuK2orN8HiM6yc0NyxWLiS8+pUMcWhSGQ9o1RzCnkGraZLJB2RyESs1n8YJk17XYKNq4/sKy0morlh8/dks9z3ZxM8CiWLfAzpu57NLBSqjEb22yezBkCCAYjl5Qxy8vmbhuRHdjsHGmkm7ZaJSWLnoYvuKTC5keEKRLSriSFFdsWjWdBBZeSRi5kBIddWj24kJegGuZ7Gx7tNpOHRbEaPjPZrNHO26z9B0hagbk69kaW90aNTaRGFEJu9RHCnw5f/+VbqtHmsXaxx6xx46jQ5xELM6v0bQi/ryowFJnHD4iX07smMBrhIne2e5aseilPp54OdF5OeVUj99G226LYRBxBc//Swq1Xk2pSFd4qJVt/orNYoz3/CoLesw+zgRXC9haDymMKQDzNaWLVVnQGEAAB14SURBVErDMWKmGKKXmJXSeiyul5IrpVh2SmtDJy2apsLoO0tTF2YPdElTLfykRCexrS4aWFZKFGnZx6BrMDoVcvjRFvMnM/h+ggjsf6BNr2dSKCQ4nuLiGY+l8zaeb1FfN2lUDS6edSmUE+yGRadhkitHdLsGjZqFYRoEgcG5Ew5eRhEHQkBCu2kzOh3g+AnnT7lM7wmIEsHxUuJYOPJ4j9e+YnF21aTb0eVJ0lRXFEiUrvYYBCZ+RvutbFshZkrQMzDdhCQWwkAouwlRaKASg3xJkSrBchSvfcWH1CJRKdO7Y60eh2itGw9SZTI23WOZESzb5tAT+0EJex7YRRKlTB+cZP1ilS995jmypQwqSbEdk2a1je05LJ9bI1PIIKaB7VhkSxlWz6/T3mhTHr37phTXwnZMxqfuPru3MhV6VkSKl+o19wvEv1cp9V+317TtZf7YBU68eAYSKFS6FMsJjaGE6mpMFBpEgQ6Tb7dMlFKghLVlmwPSJegIUWjQbrr02ibnjgsjUyGOB+2WSbdt4riKdlORzUG2oMuGJDE4jlZu67QFx9WKcs0Nk1bTYm1RC0fliinFSszqokOvY9BpCY4rjE+HlEYj0p7OkvYzKc2aQRIn+H7CykWboGtQGo7JFPptLNisLej4Fatt0q6beK6i2zLxs1pTN06EdtPg9NdzWsV/w0JS2H00oNcxWFu0aVYtgrbByHjM+K4eqws27brRz9pOWLrggNJxK62GwfB4zPBETLuZksSC7aS0Gxbryxa9jsXGOvSaphb8tgwqlaTfu5oow6FYNhnZFVHfsOi2oTjiYEqK4xuUxzK02ln2PbSbibkxROADH3sPIkISJ/zef3yG/Y/uwcu4GJbB4qllep2AVrWNShVBp0cSJ5THijRrbWYOTtKstu/0V/KGiMKEpQs703n795VSv31pRym1ISJ/H50YuCNp19tcPLFIp9lhZCrggadahCE0qiaihDQ2qK059HoJKoE0FS4V53nhczkm5iLmDnQplBPWlmxWL9isLpkceqhLeTTB8xO6HYXt2PTMhEJZh7nXVgw6HZulszapEsIuVCZCHFc483Wfk694WHbK5O6IJNFlOr7yhwXKIzEj4xEqFVbmHbqX6hKZKY6bUh6JGJlIOHfSZuG0w9h0QGk4ptc2tERkTY+gkhRSpcjkY9aXLE687FEcSjn7ms/KgkMmF+N42k/UauRxfa3voqUaDOrrNmJGeFlFcTjCcRNKwynL5x2WzrkMT2hnriEG5153aVRNhid0mxvrFqdf9ZncneBnQ13qFUhjB5UmoFKCrkkSWTiehWVbOLbBroMGa4s+KF1Yzna6rC9mmdo3wczBSaqLNR791jdV+nvtHkmU4GUcvKxeJh6bHWb+6xdorDfJV7IYhonj25RGCpTHyximge3tUIU3xY6NY7lSXckdXZp1bbGG69lgJkzO9TBMqC07dNsWubJ21Pa6BkHX7qfdC6mCbsek1zaIQ4Pqikk2H5EvKtxswvkTGeprNtlizOSsycTuiKV5YeZAD9tMUAimDSsXLc4c87lwSi9V58ox972jwfqSRRyZBIHJygWhvm7SbQkqESojMX4uxXJCGjWTVtMkV0y4cNph95EuI5MpjSpk84oH39Vm3/2d/ogkxbRTnIxBqyHU1x09lSvHnDnmUV12yBZ7GKbCclOS2KTdSHA9SCKT+RMOdiZFJXqaMjQeYruqXylRR/t+4wWPfBGKFYXvJ9RXXYpDMa26Q7PqsLbk6pB7II5MWnVFZdyiXs0TBRHDEzGlEYOg43LhtJBEusKASEqnUyKTy7LroEtlYozCkMXpl5fpdPYxNDlCdanG3P272P/omys6pm2RK2eoLdVxMzpy2st6DE0N4Wddpg5O0ql3GJsdQUSIo5jmemtbpQ22E9sxGZ++tqzD7WYrHcTzIvJP0SpTCvhR4KvbatU2o5IUQXCMCNtN6LYt6lXtQIxDkzQ16ee1sTn7wLIVBx7sUBnVEa9xbNBqGdSXbAwzZfZgwORcQKGSkivE1NYsPE9Rq7pEPSEM4PUXfdaXbeo1G8tOGS9EdDs2rpcytSegvmaysmAxMgEb6yYjkyHVZZvF8zaWoaNsdx8KSBLIFRLWFh1GxjuMTSfM7O9RXzeIQpOgZ6AQMvmE1QUbUos4EgzRsSqZnGJyd0gS6hWwQjHp69tqoadMLiHqWdRWTYZGE6LAIEqFjVXB97WebxwK88c9zp/IAIqpPQGFckQcCUoJylC4ni5DYlpge6ZeRRPRkppdk1bdYWO9QJqaZItV0hgqEwlDk1m6nUnOHo+ZPeoxe/QwYhaZfXCOJDbYWK4zPF1h7uiut4hVexktsN2qdWhWW1i2Ra8TkMYp7/rwE5TGipz/xkVWz69zO6UNtosojFk6f8vzc2+arXQsPwr8XeBT/f3fB3522yy6DZTHSpx48TStep2puYi1JYeNdRvDUESREEdXft3hR1vc/1QHP5ti2braYG1Vd0ql0Yh8MWXpgk2x0sXLpewZ7pGinZuGBUlHGJ6ISCKT9eWUmX09TEt46YtZpveEKKAynpAq/YN1/YRMHpSkGGKSIjh2ikoUlqNwXIXjJYgp7D7cIltUvPZchvlveASBHqVEocXoVIQYsDjv0G5ZGFZIzlf0usLyRZ2JbVkpYzMhnZYW4O51TDrthIVTLhsriuHxmE5TdJJjPmVtxSLuWew5EiESsrbosLboUB6NiALdMSkBxzWwXYN82SaKC1i2w8K80Ot2yZc8nOwQthLCbkxxeBIvF+P6KfPHI5xsjskDZR5874OUJibIl3Msnlkm7EX4OY/acoNO43UOvWPfW6Jjd9+3C9MyuXB8gU6jQ2msyH1/7hCVcf3PPnd0F5P7Jkii2yNtsJ3YjsX41A4csSil2sBPiUhOKdW6DTZtO4WhPFGvQa5YxxBFJpsQBUKvZxJ0jSsu32XyMQ882cbLKDpNgzgymNrTY2Z/j6XzDqIEN5vSahocf9XjkBEyNhWQJAb1mklt1SYOobpq42YSTNElOs696tPracGnXCnBq2p/xMK8w/h0SK6UkskrPD+msWERhVqPVgTSRCiWFRurJq2mrqWcRoqp3QELZ13tlzBDwODs6y5eRjt0l855ejVLUspDEVFPWF92MB0olGLiOKFZtamuuLg+nHrVwvYSJmYDCqWUetUmioTGuoVhwvTegOqydhyfeCnD+GyP8Rl9XjF9/HyWVI2zch78fJG9j+Sor9WZOTRNbXGDkV1DiAjjcyPYrk1hKM/rz51m3yNzZAoZlFJ4GZdOs0MSJ5T6urIZoFltsXh6mbmjb0oEWbbFnvtn2XVoijRJsV37mxT4HdcGd4f6VTYRhTFL53ZQHMslROSdwC+h06Z3iciDwA8ppf7Gdhu3XYgIveZpwo5i6YJNfd0hDKDXunrSdWU0JkmEJIZex6Q8oqNxk1jIFlLWFiw6bYuRiYRjX83wzCmf9//FGl5Gsb5kYYhi6YJDs2ohBhx/2cfJpAT9KVfQM4iWTHLlGOkLa8cxZLIJrq+Xgz1f5+OkSnHuNY+ZfSFxItiWIgrg/AmXlSWXidmAR97d5PWXfZ3v1IaRyR62IzTWbRo1E9eL6XVsKqMxbiZleDIkiYT1FT0tU4YgYmA5CfWaRbvuk6Ym1UyCiIubNchVIOpGWE6Km9XRvnFscfF0nvUVi9EZh733z2JnymRyRZxsk4Pv2E9pOE/Yizj27EnEEKqLGxx4dDcq1cLmp1+eZ/f9M0zuHQe0RvDFk4tEvYjhyzKNs8UMaxerb+lYLmHZFuz8vuPtUWiFvLuMrUyF/hnwbcB/A1BKvSQi795Wq24DabhCEitOH8vQbVz7Mli2LotxSabSz+oiyoYhdJsGiBAFOm7DzyWsLfqcPeERRwrbNlCiyGQUzQ1FbdUhiYVeVygUE4JAcJyUXsfA81I2aqZW3zeE2rJFcUTLSTpeQhIKrz2fZWgkorZm0awnzB1MaNVNLpx2OPWyz8mXPEanIhBYX3SIIyiPJ3h+TH3dws8luJ4iX45ZPmdTXbXI5BSHHunRadjYvsDXDZTYFMsKw/awfSEKPSZnDUpjRdaXali2hSCYlsLxfUzLQKmUykQZL+cye2SGvQ/tobHWYPbwNF7OY3iq8sY1HZ4e4uxr51k+s4JhGlQmSnRbAaZtMr5J1d7o1+lpVtskcYrhvLmekCTpPVezZzO2YzI+U7n2gbeZLa3uKKXOXzaUTG7mpFuoY+Kiaw89ilaO+6hS6uzNnPNyqisR7Q5b6lQA7UtZt8gWU1wvRQy9etHr0o90tdl3f1cH11mKUiVm+ZxDHAq9romfS+m2BMdN6TQhm4+5eNpl79EexUpCtpAS9RKWzju0GkIurzCdhNPHfLxzioOPtGnUTM6f9CHVSZBuJiGXM2nWIn7n1yaIYyiWU50BveISRYrFMy5+TpEt9Fhb8Kitulh2yrm6x+FHO2SKFp2OQ5oqqqsGuw4EbKwaeFmDqf0VluczlEdNDMPAsHqM7e5gWRaJGiXstMjmQ2prk5TH81imEHQjdh2e4vA7DnLwHXsJuiG9yTLv+K5HOPH86bcU+soWMkzuHeedH3wML+PRbXVJkxQ3Y2M7b/1cVKoYmx2mVW1RGitqnRylaNVazN13U0qpOxo9FdqZztvz/emQEhEH+FvAsRs94RbrmPxVoKaU2iciTwO/wDXqmFwPqxfX+N1fPkG7vfVV80bVZP51t188TCv3Z7MJ9arJwhmHZl1LPSoUjZrFxVMO1TWffUd7lEYTlFJksoogEOJYlyitrxq8+uUshaGEXEGr1YmhA+oy+ZReR3AzCi+T0KhabKya+H5CppDoSoixkCsHfPa/DrOylKFYVqytQKbUoLHmYFoK19NiTyc7FvlSiu1aGAaMTjssX/BpNyCTU6SpEPYSFs74rC35mFaObOUJhkWws03GZkdI4oRMNsE0L5ApLNNt5rCzD3L/++8jTbVebNSNmNo3TmWiTJroUd2hJ/bjZzzm7pvh9edO0muHWjC8G1EczjM8OYRpmRSHCyildOJhq4ef0yH2SayTE/c8OMv6Qo2lM6t98Q6Y2DPGyPTdK8S03diOtWNHLD+MHl1MoTUxfw/4mzdxzmvWMenv/1x/+9PAJ0REbpU85UvPvEareWUn7dVQSjj1ql4qntnXI1tIef0lj0IpJVeKGJoMWVsxefXLOZbPuZi21ht57fkM+UJCYTjBcROSSLg475DGguVo7dt23SBNLEwnwVBCaVT7LYKqjZ+LcVzotg1sL+Xwox1yhQRMSEKT+ZNZUplm34M+k7ua1KqCY1o8+R0maeqRYpOkBYL6CqdeNcmVHA49Ps3CuYC1c10gZmTOYHyXx8qFiNQcozhuY5gmtTVIgojKRIlswWf3A7P02gFrF4dptzYYO1BhZHoYP+eRJimWa3PwsT3EYUJteUNPb8ZLZItaka1QyXPftxyhulgj7IUUhvIURwpvWZUREfY/vJvXnz/NxmodQUCE3Q/Mki1kyRayjO8eI+qF2J6jnbD3MFEYs3T2lmrM3xK2siq0BnzsFp5zK3VM3jimr8lZB4aAb7qCN1JXqFPvgDK4TDn7migl1FZsaitvfplFUvJlPTMMAtFCSqYi6PT9AFZKlOol7CQxaVRNLFPwijG+r+h1UspjEaNTIbUVi+qSQ23ZZvGsjsIVEQwjpVW38LKC49gMjSfEkc/onse5788fZuZoi3wli+sHDI2t4XomjtOlOGKhKLF2scHaylHe9fSDKCU0qy32Pe4yvmeUykSJpdOrLJ5eYni2y+r5NUZ3j5HN+6ycWyVTyDA6M0S73sEwtGyj4zk8+N772H3/LtobHdr1Do7vUBzOvxFTUhjKX/Eaehn3Dafs1fBzPve/+zDteoc0SckU/Lf4URzXvuc7lLeyA523IrIHPWJ5Ev0f/6fA37k04rgBtlLHZMu1Tm6krtDeh3df5RTXj1IGjeoVgpPNlGIxwvMV2XJCFAmLZ13iQP87dzs2YS6hUElprLusLmYYnrQZ36szk83IpzIxjlfwadcb7HvIIlExjapicdHlHd/6CN/7kx9CDIMzr5xj+ewyhmkyvvspZg4M06oHLJ9dIQ47TB8d5bHvmnjDt3Fp4HfJb1asFNj38BxREL9RvzhNEh791gfIFjKATtqsr9aJw4RcOUuulEVEKAzlr9qJ3AyGYezYoLXbiZ4KDd9pM76JrUyFfh3tE/lwf/9p4JO8fbW0t2MrdUwuHXNBRCygCNyyxfqDj+/jfd/3Tj77yT/hkhL/jaHD/d+oEvZGZ5WQr0TM7A8YnUqorXmsnPcY31OgNFxibK7C3NEZ7n/XEcbnRnGyw6xdqLF06iRpnDC2Zxf5yihJFOPnM8RRzMvPvMaF44u4eYcjTx7kyFMHsCz98R18bC/7HpoDdEkTADcDQxNX1mq9PKYDdE0cM6Nfm8l9czF0x7UZmb77vsD3OlEQsXR2SxU5bitb6VhEKfWrm/Z/rV8u4Ea5Zh0T9NL2/4oeHX038Me3svyHZVl8/N//CHsenuWXf+KT6BKcl5q/0mkSdKchYCpILfBtRiaKjM+O8+S3P87DH7iPsblRGqtNSGLECgmDFNvJkC34FEfK31RcfDP5YpndR6+uYvb+j739Cv+lDmXAvYXtWIzP3n0d/lY6ls+KyE8Bv4H+1X0U+B0RqQAopa5rJLHFOia/DPyqiJxEj1Sevp5zbAXHtXn64x/h6Y9/5Ja2Wyjd+mnBgAFXIwpjvUp2l7GVjuXSMu8PXfb4D6I7musWC91CHZMe8D3X2+6AAfccqi9ZehNsR1zZVlaFdt+owQMGDNheLNdifO7G6x5tV1zZVSf9IvJ4v4bypf2/LCKfEZF/eWkaNGDAgLsApa5+uzZvxJUppUK0y+MvXHbMXwD+Y3/708D75UorAJuQq/lEReQF4ANKqWo/N+g30BIKDwGHlVLfvRWrbycisgrMX+fLhrlCfMxdzk60GXam3XfC5lml1JaGISLyP9E2Xg0P6G3a/3f9EI1Lr/9udLH3v9bf/0vAE0qpH9l0zKv9Yy7090/1j7nqdXm7qZC5yTH70b5B/wX4L5uKud9VbPXD2IyIPK+Uemw77NkudqLNsDPtvtttVkp9+002cUvjyi5x9fVPMPsxJADvB/5403M7WppywIABb3A9cWVsNa7s7TqWTwKfE5HPAF3gC/2G9wH167F8wIABdy1vxJX1k4yfpi+RsolLcWWwxbiyt6sr9H+IyB8BE8Dvb2rIQPta/qywE4uv7USbYWfavRNt3jLbFVd2VeftgAEDBtwobzcVGjBgwIAbYtCxDBgw4JZzz3QsIvLtInJcRE72c58uf94VkU/1n/+KiMzdfiu/yaZr2fxuEXlBROJ+PMIdZws2/7iIfF1EXhaRPxKR2Tth5+Vswe4fFpFXRORrIvJFETlyJ+zcMSil/szf0E6pU+i8Jgd4CThy2TF/A/i3/e2ngU/tAJvngAfQeRzfvUOu8/uATH/7f7vT1/k67C5s2v4g8D/vtN138+1eGbFsS9jyNnNNm5VSZ5VSL3P3SIhtxebPKqU6/d0vo+Mm7jRbsbuxaTfLdQmb3nvcKx3LleQwp652jFIqRsfq3EmV5q3YfLdxvTb/VeB3t9WirbElu0Xkb/bD2f9PtKj8gKtwr3Qs2xK2vM3cbfZshS3bLCLfDzwG/JNttWhrbMlupdS/VkrtBX6SHV5meLu5VzqWbQlb3ma2YvPdxpZsFpEPAD8DfFApFdwm296O673WvwF8aFst2uHcKx3LtoQtbzNbsflu45o2i8jDwP+N7lTuFrHWrdi9f9PudwEnbqN9O4877T2+XTfgO4HX0d7/n+k/9g/RX3DQ6eW/CZwEngX27ACbH0f/27bRyl6v7QCb/xBYBr7Wv/23O23zFu3+F8BrfZs/Cxy90zbfzbdBSP+AAQNuOffKVGjAgAG3kUHHMmDAgFvOoGMZMGDALWfQsQwYMOCWM+hYBgwYcMsZdCx9RORnROS1ftbt10Tkif7jPyYimau85q+IyCeu8zy/cqOZyCLyS9fKqr3cXhH5HyJSupHzXaXtv9zf/p7+9UpF5KbFpkXkSzdv4Vva++FLtl7Ha5652fciIr9xWczLPcmgYwFE5CngfwEeUUo9AHyAN3NHfgy4Ysdyu1FK/TX11kJSV+It9iqlvlMptXGz5+5HI/8g8Ov9h14FPgJ8/mbbBlBKvfNWtLOpvX+rlPp/b2WbW+QXgZ+4A+e9qxh0LJoJYE31w8uVUmtKqQUR+VvAJLp+9WcBROQHROR1Efkc8Oeu1bBoPtHXIPkdYHTTc4+KyOdE5Ksi8nsiMiEih0Xk2U3HzInIy/3tN/5RReQXReT5/qjhH/Qfu5K9Z0VkuL/94yLyav/2Y5vaPyYi/0+/rd8XEf8Kb+XPAy8onaCJUuqYUur4Ft7/MyLyz0Tk8/3zPC4ivyUiJ0TkH206rtW/f2//NZ8WkW+IyH+6PMtcREZF5Kv97QdFRInIrv7+KRHJiMjPicjHN9nwCyLybP+z+5b+435/hPGyiHwK8Ded4/tE66+8KiK/0H/se0Xkn/a3/7aInO5v7xWRL/Zf+gXgA/JmhYt7kzsdoXc33IAcOqLydeDfAO/Z9NxZYLi/PQGcA0bQuh1/AnziGm1/BPgDtObHJLCBThmwgS8BI/3jPooWMqZvy57+9k8CP9vffgZ4rL9d6d+b/ccfuNzezfvouruvoFP+c+go0ofRmi4x8FD/+P8MfP8V3sc/AH70Co+/YdNV3v8zwC/0t/82OgdnAnDRUcND/eda/fv3ojPLp9F/fH8KvOsK7b4GFIAfQYfkfwyYBf60//zPAR/fZMP/1d/+TuAP+9s/vumaP9C/Do/1P6dLn7OFLn3zIWAceK5//Kf7551Cp4L8/Cbb/gB49E5/r+/kbTBiAZRSLfQP768Dq8CnROSvXOHQJ4BnlFKrSut2fGoLzb8b+KRSKlFKLfBmfaaDwH3AH4guAPezvKlN8p+B7+1vf/Qq5/le0dUqXwSOAtdSNHsX8NtKqXb//f4W8C39584opS4VofsqurO5nAn0tbkRLuXdvIJOO1hUenR4mrcm/13iWaXUBaVUiu5kr2TPl9AjxncD/7h//y30y9Rcgd/q329+f+8Gfg1AaV2bl/uPP86bn3MM/Cfg3UqpJSAnIvm+3b9+lfOuoDune5Z7e7i2CaVUgv5ne0ZEXkH/C/3KlQ69keav8Jigf2RPXeG5TwG/KSK/pU1Tb0l4E5HdwMeBx5VSNRH5FXSu09vxdqJVmzOMEzZNCTbR3cI5EJH/gB4JLSilvvOy9tPLzpVy5e/g5fZc6ZgvoH/Qs8Bn0CM7Bfx/VzHtUpuXt3e1z+Zq/CnwA8Dxvg0/CDwF/O+bjvHQ1+ueZTBiAUTk4GWe/Id4swZ0E8j3t78CvFdEhkTEBr5nC81/HnhaREwRmUBLM4L+Yo70HceIiC0iRwGUUqfQP4C/y5VHKwV04mFdRMaA79j03GZ7L7fjQ33/Qxb4MFf/d78Sx4B91zpIKfUDSqmHNnUq28Xnge8HTvRHNlX0NOdPrrONjwGIyH3o6RDoz/k9IjIsIibwfcDnNr3m4/37F9GfZ6CU2lzE7wB6qnbPMhixaHLAv+ovy8boDOe/3n/u3wG/KyKLSqn3icjPof+1FoEX0D4OROSDaF/D37us7d9GOz5fQftwPgeglApFLzv/SxEpoj+Lf86bX8hPoUWQdl9urFLqJRF5sX/sad76Y3qLvZte80J/ZHPJMfxLSqkXZeui4b8L/OqlHRH5MPCv0H6I3xGRrymlvm2Lbd00SqmzfZ/upVWpLwLTSqnadTTzi8B/6DvHv0b/2iilFkXkp9FZzAL8D6XUZ/qv+QJ6GvR5pVQiIueBb1xqsN/Rd5VSizf+7nY+g+zmAVtGRH4b+InLp2YD3kRE/g7QUEr98p225U4ymAoNuB5+Cu3EHXB1NnhTlP2eZTBiGTBgwC1nMGIZMGDALWfQsQwYMOCWM+hYBgwYcMsZdCwDBgy45Qw6lgEDBtxy/n8GUwrK4wFecgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(4,3))\n",
    "plt.scatter(df_train['std_1min'], df_train['area_hf'], c=df_train['apn'], cmap='viridis', alpha=0.2)\n",
    "plt.colorbar()\n",
    "plt.xlabel('Std. deviation (1-min window)')\n",
    "plt.ylabel('Spectrum area of high frequency')\n",
    "plt.tight_layout()\n",
    "plt.savefig('../archive/std_hf.png', dpi=600)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-06T05:10:33.428413Z",
     "start_time": "2020-02-06T05:10:17.591641Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Selected features ***\n",
      "Minute_wise AUC: 0.885, Group-wise AUC: 0.951, Group-wise F1: 0.5211425973607386\n",
      "*** All features ***\n",
      "Minute_wise AUC: 0.885, Group-wise AUC: 0.950, Group-wise F1: 0.5203431545126286\n"
     ]
    }
   ],
   "source": [
    "# Test for selected features & all features\n",
    "mdl=LogisticRegression(max_iter=1e4)\n",
    "minute_auc, group_f1_macro, group_auc_macro, res_detail = model_evaluation.model_evaluation_CV(mdl, df_train, file_train, feature_col)\n",
    "print(f'*** Selected features ***')\n",
    "print(f'Minute_wise AUC: {minute_auc:.3f}, Group-wise AUC: {group_auc_macro:.3f}, Group-wise F1: {group_f1_macro}')\n",
    "\n",
    "mdl=LogisticRegression(max_iter=1e4)\n",
    "minute_auc, group_f1_macro, group_auc_macro, res_detail = model_evaluation.model_evaluation_CV(\n",
    "    mdl, df_train, file_train, df_train.drop(['apn', 'file', 'group'], axis=1).columns)\n",
    "print(f'*** All features ***')\n",
    "print(f'Minute_wise AUC: {minute_auc:.3f}, Group-wise AUC: {group_auc_macro:.3f}, Group-wise F1: {group_f1_macro}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-06T05:28:46.864495Z",
     "start_time": "2020-02-06T05:27:57.093495Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** C = 0.01 ***\n",
      "Minute_wise AUC: 0.887, Group-wise AUC: 0.945, Group-wise F1: 0.521\n",
      "*** C = 0.05 ***\n",
      "Minute_wise AUC: 0.888, Group-wise AUC: 0.953, Group-wise F1: 0.524\n",
      "*** C = 0.10 ***\n",
      "Minute_wise AUC: 0.888, Group-wise AUC: 0.951, Group-wise F1: 0.526\n",
      "*** C = 0.50 ***\n",
      "Minute_wise AUC: 0.886, Group-wise AUC: 0.951, Group-wise F1: 0.523\n",
      "*** C = 1.00 ***\n",
      "Minute_wise AUC: 0.885, Group-wise AUC: 0.951, Group-wise F1: 0.521\n",
      "*** C = 5.00 ***\n",
      "Minute_wise AUC: 0.881, Group-wise AUC: 0.924, Group-wise F1: 0.497\n",
      "*** C = 10.00 ***\n",
      "Minute_wise AUC: 0.879, Group-wise AUC: 0.923, Group-wise F1: 0.497\n"
     ]
    }
   ],
   "source": [
    "# Hyperparameter tuning\n",
    "# Optimal C: 0.05\n",
    "# Minute_wise AUC: 0.888, Group-wise AUC: 0.951, Group-wise F1: 0.526\n",
    "for C in [0.01, 0.05, 0.1, 0.5, 1, 5, 10]:\n",
    "    mdl=LogisticRegression(C=C, max_iter=1e4)\n",
    "    minute_auc, group_f1_macro, group_auc_macro, res_detail = model_evaluation.model_evaluation_CV(\n",
    "        mdl, df_train, file_train, feature_col)\n",
    "    print(f'*** C = {C:.2f} ***')\n",
    "    print(f'Minute_wise AUC: {minute_auc:.3f}, Group-wise AUC: {group_auc_macro:.3f}, Group-wise F1: {group_f1_macro:.3f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-06T05:39:51.585748Z",
     "start_time": "2020-02-06T05:34:36.384474Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Layer size: (16, 16) ***\n",
      "Minute_wise AUC: 0.891, Group-wise AUC: 0.925, Group-wise F1: 0.522\n",
      "*** Layer size: (32, 32) ***\n",
      "Minute_wise AUC: 0.898, Group-wise AUC: 0.948, Group-wise F1: 0.525\n",
      "*** Layer size: (64, 64) ***\n",
      "Minute_wise AUC: 0.898, Group-wise AUC: 0.938, Group-wise F1: 0.558\n",
      "*** Layer size: (16, 16, 16, 16, 16, 16) ***\n",
      "Minute_wise AUC: 0.891, Group-wise AUC: 0.941, Group-wise F1: 0.578\n",
      "*** Layer size: (32, 32, 32, 32, 32, 32) ***\n",
      "Minute_wise AUC: 0.899, Group-wise AUC: 0.957, Group-wise F1: 0.568\n",
      "*** Layer size: (64, 64, 64, 64, 64, 64) ***\n",
      "Minute_wise AUC: 0.892, Group-wise AUC: 0.937, Group-wise F1: 0.560\n",
      "*** Layer size: (16, 16, 16, 16, 16, 16, 16, 16, 16, 16) ***\n",
      "Minute_wise AUC: 0.890, Group-wise AUC: 0.945, Group-wise F1: 0.543\n",
      "*** Layer size: (32, 32, 32, 32, 32, 32, 32, 32, 32, 32) ***\n",
      "Minute_wise AUC: 0.881, Group-wise AUC: 0.953, Group-wise F1: 0.574\n",
      "*** Layer size: (64, 64, 64, 64, 64, 64, 64, 64, 64, 64) ***\n",
      "Minute_wise AUC: 0.884, Group-wise AUC: 0.957, Group-wise F1: 0.566\n"
     ]
    }
   ],
   "source": [
    "# Hyperparameter tuning\n",
    "# Optimal structure: 32 x 6\n",
    "# Minute_wise AUC: 0.899, Group-wise AUC: 0.957, Group-wise F1: 0.568\n",
    "for layer_size in [(16, 16), (32, 32), (64, 64), \n",
    "                   (16, ) * 6, (32, ) * 6, (64, ) * 6, \n",
    "                   (16, ) * 10, (32, ) * 10, (64, ) * 10]:\n",
    "    mdl = MLPClassifier(\n",
    "        hidden_layer_sizes=layer_size, \n",
    "        max_iter=2000,\n",
    "        early_stopping=True,\n",
    "        validation_fraction=0.1,\n",
    "        random_state=123)\n",
    "    minute_auc, group_f1_macro, group_auc_macro, res_detail = model_evaluation.model_evaluation_CV(\n",
    "        mdl, df_train, file_train, feature_col)\n",
    "    print(f'*** Layer size: {layer_size} ***')\n",
    "    print(f'Minute_wise AUC: {minute_auc:.3f}, Group-wise AUC: {group_auc_macro:.3f}, Group-wise F1: {group_f1_macro:.3f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Light GBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-06T06:32:27.442751Z",
     "start_time": "2020-02-06T06:01:39.115988Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_auc_macro</th>\n",
       "      <th>group_f1_macro</th>\n",
       "      <th>max_depth</th>\n",
       "      <th>minute_auc</th>\n",
       "      <th>n_estimators</th>\n",
       "      <th>num_leaves</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>50.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>100.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>500.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>50.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>100.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>500.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>50.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>100.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>500.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>50.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>100.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>500.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>50.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>100.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>500.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>50.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>100.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>500.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>50.0</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>100.0</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>500.0</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>50.0</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>100.0</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>500.0</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>50.0</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>100.0</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.956576</td>\n",
       "      <td>0.566218</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.884164</td>\n",
       "      <td>500.0</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    group_auc_macro  group_f1_macro  max_depth  minute_auc  n_estimators  \\\n",
       "0          0.956576        0.566218       -1.0    0.884164          50.0   \n",
       "1          0.956576        0.566218       -1.0    0.884164         100.0   \n",
       "2          0.956576        0.566218       -1.0    0.884164         500.0   \n",
       "3          0.956576        0.566218        8.0    0.884164          50.0   \n",
       "4          0.956576        0.566218        8.0    0.884164         100.0   \n",
       "5          0.956576        0.566218        8.0    0.884164         500.0   \n",
       "6          0.956576        0.566218       16.0    0.884164          50.0   \n",
       "7          0.956576        0.566218       16.0    0.884164         100.0   \n",
       "8          0.956576        0.566218       16.0    0.884164         500.0   \n",
       "9          0.956576        0.566218       -1.0    0.884164          50.0   \n",
       "10         0.956576        0.566218       -1.0    0.884164         100.0   \n",
       "11         0.956576        0.566218       -1.0    0.884164         500.0   \n",
       "12         0.956576        0.566218        8.0    0.884164          50.0   \n",
       "13         0.956576        0.566218        8.0    0.884164         100.0   \n",
       "14         0.956576        0.566218        8.0    0.884164         500.0   \n",
       "15         0.956576        0.566218       16.0    0.884164          50.0   \n",
       "16         0.956576        0.566218       16.0    0.884164         100.0   \n",
       "17         0.956576        0.566218       16.0    0.884164         500.0   \n",
       "18         0.956576        0.566218       -1.0    0.884164          50.0   \n",
       "19         0.956576        0.566218       -1.0    0.884164         100.0   \n",
       "20         0.956576        0.566218       -1.0    0.884164         500.0   \n",
       "21         0.956576        0.566218        8.0    0.884164          50.0   \n",
       "22         0.956576        0.566218        8.0    0.884164         100.0   \n",
       "23         0.956576        0.566218        8.0    0.884164         500.0   \n",
       "24         0.956576        0.566218       16.0    0.884164          50.0   \n",
       "25         0.956576        0.566218       16.0    0.884164         100.0   \n",
       "26         0.956576        0.566218       16.0    0.884164         500.0   \n",
       "\n",
       "    num_leaves  \n",
       "0          8.0  \n",
       "1          8.0  \n",
       "2          8.0  \n",
       "3          8.0  \n",
       "4          8.0  \n",
       "5          8.0  \n",
       "6          8.0  \n",
       "7          8.0  \n",
       "8          8.0  \n",
       "9         16.0  \n",
       "10        16.0  \n",
       "11        16.0  \n",
       "12        16.0  \n",
       "13        16.0  \n",
       "14        16.0  \n",
       "15        16.0  \n",
       "16        16.0  \n",
       "17        16.0  \n",
       "18        32.0  \n",
       "19        32.0  \n",
       "20        32.0  \n",
       "21        32.0  \n",
       "22        32.0  \n",
       "23        32.0  \n",
       "24        32.0  \n",
       "25        32.0  \n",
       "26        32.0  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Hyperparameter tuning\n",
    "# Yielding the same results?\n",
    "# Minute_wise AUC: 0.884, Group-wise AUC: 0.957, Group-wise F1: 0.566\n",
    "para = {\n",
    "    'num_leaves': [8, 16, 32],\n",
    "    'max_depth': [-1, 8, 16],\n",
    "    'n_estimators': [50, 100, 500],\n",
    "}\n",
    "res_df = pd.DataFrame()\n",
    "\n",
    "keys, values = zip(*para.items())\n",
    "for v in itertools.product(*values):\n",
    "    experiment = dict(zip(keys, v))\n",
    "    gbm = LGBMClassifier(\n",
    "        boosting_type='gbdt', **experiment, \n",
    "        random_state=123, n_jobs=-1, silent=True)\n",
    "    minute_auc, group_f1_macro, group_auc_macro, res_detail = model_evaluation.model_evaluation_CV(\n",
    "        mdl, df_train, file_train, feature_col)\n",
    "    experiment.update({\n",
    "        'minute_auc': minute_auc,\n",
    "        'group_f1_macro': group_f1_macro,\n",
    "        'group_auc_macro': group_auc_macro,\n",
    "    })\n",
    "    res_df = res_df.append(experiment, ignore_index=True)\n",
    "\n",
    "res_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Comparison using cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-06T16:25:19.040126Z",
     "start_time": "2020-02-06T16:25:19.028739Z"
    }
   },
   "outputs": [],
   "source": [
    "# Define and CV evaluate models with the optimal hyperparameters\n",
    "logreg = LogisticRegression(C=0.05, max_iter=1e4)\n",
    "mlp = MLPClassifier(\n",
    "    hidden_layer_sizes=(32, ) * 6, \n",
    "    max_iter=2000, early_stopping=True, validation_fraction=0.1,\n",
    "    random_state=123)\n",
    "gbm = LGBMClassifier(\n",
    "    boosting_type='gbdt', num_leaves=8, max_depth=8, \n",
    "    learning_rate=0.1, n_estimators=50, \n",
    "    random_state=123, n_jobs=-1, silent=True)\n",
    "\n",
    "mdl_res = []\n",
    "for idx, mdl in enumerate([logreg, mlp, gbm]):\n",
    "    res = model_evaluation.model_evaluation_CV(mdl, df_train, file_train, feature_col, plot_roc=True)\n",
    "    mdl_res.append(res)\n",
    "\n",
    "with open('../archive/Model_Evaluation.pkl', 'wb') as f:\n",
    "    pickle.dump(mdl_res, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-06T21:57:24.760629Z",
     "start_time": "2020-02-06T21:57:24.754612Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** logreg result ***\n",
      "Minute AUC: 0.888, Group AUC (A vs rest): 0.953, Group best F1: 0.715\n",
      "*** mlp result ***\n",
      "Minute AUC: 0.899, Group AUC (A vs rest): 0.957, Group best F1: 0.727\n",
      "*** gbm result ***\n",
      "Minute AUC: 0.888, Group AUC (A vs rest): 0.957, Group best F1: 0.735\n"
     ]
    }
   ],
   "source": [
    "for res, mdl_name in zip(mdl_res, ['logreg', 'mlp', 'gbm']):\n",
    "    print(f'*** {mdl_name} result ***')\n",
    "    print(f\"Minute AUC: {res['minute_auc_mean']:.3f}, \"\n",
    "          f\"Group AUC (A vs rest): {res['group_auc']:.3f}, \"\n",
    "          f\"Group best F1: {res['group_f1_best']:.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-06T17:40:50.713280Z",
     "start_time": "2020-02-06T17:40:50.178931Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPQAAADQCAYAAAAwEkzeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO2dd3gWVdbAfwcSWohSAtJ7UWoogjTFQlewICCKFMFFEVEWFlZFgUVBcBXLrohSXGku8EmxIEhZRXpM6CIBgtKRGCAkIe18f9xJfJO8SSYhyZu8md/zzJOZuXfuPZOZ894y95wjqoqDg4N3UMTTAjg4OOQcjkI7OHgRjkI7OHgRjkI7OHgRjkI7OHgRjkI7OHgRjkLnQ0Skg4gcFZFIEXnQ0/I4FBwchc4AEQkTkWhLsc6JyEIRKZ0qT3sR2SQiV0XksoisFZFGqfLcJCKzReRXq6xQ6zggnaqnAh+oamlVXZUD91FNRFaKyO+WjPtFZMiNllsQsfNMb6Dc+3JCxhvBUejMeUBVSwOBQAvg70kJItIOWA+sBqoAtYG9wI8iUsfKUwzYCDQGugM3Ae2BS0CbdOqsCRzMjrAi4uPm9GfAb1a55YEngfPZKT+7pCOXp0j3mRZ4VNXZ0tmAMOA+l+OZwFcuxz8A/3Zz3TfAf6z94RjlKW2zzmNAIhANRALFMT8Wa4BwIBQY4ZJ/MrACWARcAYa7KTMSCMygzjuAbUAE5geps3V+ALAnVd4XgTXWfnHgLeBX6x7nACWttM7AKWACcA74zDp/PxBi1bUNaJaOTHOAt1KdWw2MtfYnAKeBq8AR4N4ceqYZ3VMA8KUle7j1/ItgfjBdn9nfPPbOelpp8vPm+vCBasB+4F3ruBSQANzt5rqhwFlrfxnwaXbrtY7/B/wbKIFpVS4mvcCWQscBD1ovV0k35X0H/GgpaI1UaVUxvYWe1vVdrOMK1j1eBeq75N8NDLD2Z2N+aMoB/sBaYLqV1hmIB960lKQk0BK4ALQFigKDrXst7kbmOzG9CrGOy1oKUwVoaKVVsdJqAXVv9JnauKfploL7WlsnF/lSPDOPvbOeFiA/b9ZDirReasV0ncu4vAwK3Ormuu5AnLW/AZiRjXqTXrrqmB8Of5f06cBCa38y8H0m5ZUFZmC68QmYFvJ2K20CVuvpkv9bYLC1vwh41dqvb/0vSgECXHNVJKAdcMLa7wzEAiVc0j8E/pGqriPAXW5kFkwread1PALYZO3Xs34Y7gN8c/CZZnZPUzG9hHoZPTNPbs4YOnMeVFV/zAt6K6bbBfAHpptV2c01lYHfrf1L6eSxSxUgXFWvupw7iWlZk/gtowJU9Q9VnaiqjYFbMAq9SkQEM65+VEQikjago4vMS4DHrP2BwCpVjeLPFjzI5bp11vkkLqpqjMtxTeCvqeqqbt1japkV07txrXuxlRYKvID5MbsgIstEJE0ZGZDeM83snmZhhjzrReS4iEzMQp15g6d/UfLzRtqu7+uYFzrpeCvpj6EXWfvDMWNIv+zUi/sW+g1SttCLsnhfTTCtU3nMhNDHGeT1wXTxA4GfgR7W+SJAFFA1nes6A6dSnfsIeDkLcraw6q5p1VXGTZ6bgKWk6mVk55lmdk+pymmM6SUkDX1O4LTQBY7ZQBcRCbSOJwKDReR5EfEXkbIiMg3TTZti5UmaYV4pIreKSBERKS8iL4lIz8wqVNXfMJNH00WkhIg0A57Caq3sICJvikgTEfEREX/gGSBUVS9hutQPiEg3ESlq1dFZRKpZ9cdjJt1mYcaVG6zzicDHwDsiUtGqp6qIdMtAlI+BkSLSVgx+ItLLksndvQdjFPoT4FtVjbDqaSgi94hIcSAGM7ZOsPv/SEXyM83snkTkfhGpZ/Vsrlh1JtV7HqiTTRlyDEehs4CqXgT+A0yyjrcC3YCHgbOYrnALoKOqHrXyXMeM9X7GKMMVYBemm7fTZtWPYSZ+zgBfAK+p6oYsiF7Kui4COI5p8Xpb8v0G9AFewijPb8B4Ur4bS6x7WG4peBITMF3QHSJyBTP51jA9IVR1D2Ys/AFmyBIKDMlE9qVW3UtczhXHzAn8jun9VLTkR0QeFxHbn/xSP9NM7qm+dRwJbMf0zrZYadOBV6yu+ji79ec0STN0Dg4OXoDTQjs4eBGOQjs4eBGOQjs4eBGOQjs4eBH5acF8lgkICNBatWp5WgwHhzwnKCjod1WtkPp8gVboWrVqsWfPHk+L4eCQ54jISXfnnS63g4MX4Si0g4MXkScKLSLzReSCiBxIJ11E5D3Lk8c+EWmZF3I5OHgbedVCL8SYFKZHD8yyuvrA0xgzOwcHhyySJ5Niqvq9iNTKIEsfjIcPxayhLSMilVX1bF7IV5A5fyWGF5aFEBWX1jZBNJFxV2dQMeFCjtQVI8qB4okEl0jkeLFEnEXDOUuJmATuCL7CyQ6N+efwddkqI7/MclclpU3vKetcGoUWkacxrTg1atTIE+HyM4fPXmH78Uu0qFGGm0r4pkgrlRhJx9+3csanOr/7VMpy2YpyxieBvcWvs7d4LD8XiyVeoHgi1InzxVclp26j0FPtdDRj5h6n0oXrvFe3QbbLyS8K7e7NcNsAqOpcYC5A69atnUbCYtL9jWhZo2zKk9ER8CZUufdZqrR71lY5UXFR7Di7gx9P/8jW01s5c8207nVursPAqh3pWLUjrW5pRbGixXL6Fgo3GzbAvCGwcREv3H13tovJLwp9CmPIn0Q1jKmgQy6jqoRGhLL19FZ+PP0jQReCiE+Mp5RPKdpWbstTTZ+iY9WOVCmdFYcgDraIioJNm+D++6FLFwgNhZIlb6jI/KLQa4DnRGQZxoHc5cI2fv56/1lWBZ92m1b9+lF6RiymiCamSSsZn8hHvnHU+W4hlErZamr8dWaWK8OZ0+tg05E01yaSyM/hP3Pu2jkA6pWpx6DbBtGhagdaVmyJb1HfNNc45BBHjkDfvvDzz3D0KNSqdcPKDHmk0CKyFOOSJkBETgGvYbwmoqpzgK8xXidDMS5ghuaFXPmJz3f/xs4Tl6hV3i9NWqeY72kV8wMnitRyOw65pbhQOjoarqccuVxFWXTzTVSI+4Oyke6Vs3H5xvyl2V/oWLUjlfyyPs52yAbLlsGIEVCiBHz5pVHmHCKvZrkfyyRdgVF5IUt+pmGlm1g9qkPahC3bYAvUnhQCkoWJqNgrsLQDQ5s/zaBGg3JMTocbYMwYeO896NDBKHa1ajlavLNSzMEhL6lVC8aPh82bc1yZIf+Mob2CxERl9saj/HEt1m16y0trqRL9i9u03uFRlCxWFL76vzRpYWd2sbRcWRJ2vp4leeIS47KU3yGXWLUKfH2hVy948cVcrcpR6BzkZHgU7208il+xohT3LZom/a8JH1CCWKIpkSatAVAsvggcTHvdN36+LLnZnzJh3yJuv/ClT4WSFWhQNvvfNR1ugLg4mDgR3n7bzGL37Jm1IVM2cBQ6B0lyuPjGw03pE1g1bYYZPtDscUr0nJm1gkM+hL3/5vv+3yO5/EI45BC//Qb9+8P27TB6NMyalevKDI5COzjkPKdOQYsWEBsLn38O/frlWdWOQmeRM1fCmfq/BcQmpB0nR8cmUKz8H2w5f5jz+25KmZiYCH6+cC0U9s3NUp1B54NuRGSHvKZqVdMqP/YYNMjb4U6WFVpEKqpqzqz2L4B8EvQ1P4YvTDe9eEX47pzZ0nBTSYg6CsFHs1xvDX9n3Xq+5uxZGD7cdK0bNYLXXvOIGLYUWkRuBt4H+mFCf/iJyANAa1X1jOQeIiHRWDW932kpbarXT5MugE9R62vgxV9gw2sQugHK1oL7psKtPbI1lioqRZ3xc35l0ybTGkdGmlVfjRp5TBS7LfSHmDCb9YF91rmdwD8xq74KHSV8fCnlW9x9YlQ4bJkOu+dBsdLQdRq0eRp80snvUDBJSIDXX4fJk+HWW41iN27sUZHsKvR9QDVVjRURBVDVCyJyS+6J5kEiL8DRDbgz+Cp/yTglLBn6JVzanfbaq2dh2/tw/Sq0Ggp3vwR+AWnzORR85swxXesnnoAPP4TSpT0tkW2FvoKJPJg8MhSR6piIe97H1tmw419uk2qXLgUVAii37Q2Ij3ebh7r3QNfX4RbPdb0ccpHr16F4cTNmrljRGFnkk+GQXYWeDywXkZeAIiJyOyba3ke5JpknSbgOJcrAyB/SJO3d/n9wYR5nH/4/qlepnfbaIr7gXynfPGCHHCQxEd56Cz75BHbuhLJl4dFHPS1VCuwq9HQgFpgHlMCE9vwIeCeX5PI8RYpCmbQzy3HFjBOBxNKV3KY7eCnh4TBkCKxda1rkomlX9OUH7Cp0eVV9C3jL9aSIBGBi9HoV1xLj2VusCJzelibtfExY3gvk4Fl27TKLQ86cMZZSzz2Xb3tgdhX6OHCTm/O/YMbWXsVH0cdZULYkfPcXt+mqQgmfGzdGdyggTLJiwW/dCm3aeFaWTLCr0Gl+jkSkNJDWhYYXEKUJ+Ccq/+61KE3aliMXeHfDGcoWL+8ByRzyjMuXzdLNChXgP/8x1lLl8n/blaFCi8gJzLebkiJyPFVyALAytwTzND5AYMXANOePnzqFxjpmiV5NSIiZ7KpbF9atg1sKztfZzFro4ZjWeQ0wwuW8AudV9aDdikSkO/AuUBT4RFVnpEqvAXwKlLHyTFTVr+2W7+Bww6iaGezRoyEg4M+udgEiQ4VW1Y0AIlJJVa9ktxIRKQr8C+iC8fC5W0TWqOohl2yvAP9V1Q9FpBHGz1it7Nbp4JAlrl2DZ56Bzz6Drl1h0SLT3S5g2BpDq+oVEWkCdMJ0tcUlbaqNItoAoap6HMDy7tkHcFVo5c+Jt5tx3Pg65CUxMWbSa+pUeOmlfPtZKjPsGmc8hTHO2IhpZTcA9wJrbdbjLjJG21R5JgPrRWQ04IdZbupOFidyhkPOsXYtdOsG5cvDgQNQqpSnJboh7DoJnAj0VNUHgGjrbz+MwYYd7ETGeAxYqKrVMC59PxORNPKp6lxVba2qrSvkVpfIkiwiKjbNdi02bQwphwJIdLRxpdu7N3xkLXgs4MoM9j9b3aKqW6z9REvRvgL+Awyzcb2dyBhPYUWoVNXtIlIC073Pc9vr8KjrJKIETt2Qbp5kE0mHgsfRo2a11759pnv9zDOelijHsKvQp0SkpqqeBI4CvTArxOx+v9kN1BeR2sBpYAAwMFWeXzHd+IUichtmielFm+XnKPEJCkXhtQfcG1eU8ytGlZvTOvpzKAB8/TUMGGC+K3/9NfTo4WmJchS7Cv1PoAlwEpgGLMdEvhhr52JVjReR54BvMZ+k5qvqQRGZCuxR1TXAX4GPReRFTKd3iCZ53fMQQzu4Mb5wKNhUr25We82fD144B2N3lnuey/6XIlIWKK6ql+1WZH1T/jrVuVdd9g8BbsJGODjcICdOGGd9EydC06bw3XeelijXyNZAUFVjAB8RmZ7D8jg45Cxr1kDLljBjhnGt6+VkqtAiMlhE3hGRZ0XER0RuEpFZQBjQMtcldHDIDnFxJuRMnz5Qpw789JPpbns5ma3lngkMArZhPivdAbQDgoCOqro31yV0cMgODz5oJr2efRb++U8T6bEQkNkYegBwp6oetWaeDwKPqernuS+ag8MNMGIEDBpkZrQLEZkpdBlVPQqgqodFJMprlPm7yXAgbWC430UJLxGHFNClf4WWhATjffOWW4wDggcf9LREHiEzhRbLGWDSSq/4VMeo6q+5JVyucmwzJMRC7bsAuK6JLIo9w8exp4lRaPZHWp/bDvmUc+dg4EATonXkSE9L41EyU2g/zOSX69LNky77ivmuXDCp1Ax9aA4bTm7g7aC3OR17ms7VO+PzR2/+5/0Tot7Bli3Gyf3ly7BggfH7VYjJTKF980QKD3FQo5m5bgg/XfiJ+mXr83HXj7mj8h1MWnUAOOtp8Rwy4+RJE6a1bl3YsAGaNPG0RB4nM3tor7VEWFA0mncSwil75QqvtnuVh+s9TNEiBbezUaiIjYVixaBmTVi2zNgv+/t7Wqp8QaG1MFhXNJZbKc6XD33Jow0edZS5oLBtm4nouHmzOX7kEUeZXSi0Cg1QQXzxL+a8DAUCVXj7bbjrLuN84OabPS1RvsSJD+2Q//njDxg6FFavhoceMoYVZcp4Wqp8ie0W2lr22U5E+lrHJUXEcU7tkPt8/jl89RW88w6sXOkocwbYdUHUGFhtHVYCVmBslx/HLAl1cMhZVCEsDGrXhr/8Be6806NxlwsKdlvoD4FpqlqPP50abME4DXRwyFmuXDFLNlu2NOFnRBxltondMXRTjM9ssDxuqWqkiBR8J0wO+Yt9+4x7oOPHTTD1SpU8LVGBwm4LfRJo4XpCRFoDx3JcojwgPjGeGDfB3AFUlcjr6cR9dsg9VM1kV9u2EBkJmzbBhAlQpFB/iMkydv9brwJficgkoJiIjMeMo1/N+LI/EZHuInJEREJFZGI6efqJyCEROSgiS+yWnRW2nt7KI2se4XiRRBqT0qTuyLmrPDl/F18En6ZpVeezSJ6zZQt06ADBwWbM7JBl7LogWiMiZzHhcH4EGgL9VHWXnevtRM4QkfrA34EOqvqHiFTM2q1kzLGIY8zaM4sfT/9Idf/qvBtbmrtLBgBwKfI6b2/4haW7fsW/hC+v3t+IQe1q5mT1Dulx+LAZI996q3GnW6xYgXVynx+wO8tdVlV3Y7x3Zgc7kTNGAP9S1T8AVDVH3Pf+EfMH/w75N8t/WU4pn1KMaz2Ox259jGKfdCEBmPf9Md7fGEpUXAJPtqvFmHvrU9avWE5U7ZAZS5bA00/D7beblV8lna+gN4rdSbHTIvIdsBhYo6rRWazHTuSMBgAi8iPGgmuyqq5LXVBWImfM/mIcS69sIpo47qI2veNvw/+nY/z00zSaXDrDvgs+vHHgZ+5uWIGXe91GvYrOqrE8ISYGXnjBtMgdO5o4Ug45gl2Frg30B17EuNpdDSwB1ts04LATOcMHqA90xjji/0FEmqhqRIqLVOcCcwFat26drpvf2NjrfHp5Hbddj2Xq7+HUi/sV+F+KPBeKteHTYW24q0HBC0pWYDl7Fnr1MuPkCRNg2jTwcRYs5hR2x9DngfeA90SkDsZJ/luYyBZ2gufaiZxxCtihqnHACRE5glHwbHXzEzWBeBHq+DaiyphPiXKTp08pf8SZRc1bypUzUR3XroX77/e0NF5Hdt7mm63NH/uxrZIjZ4hIMYyvsjWp8qwC7gYQkQBMFzx1kPmsI0UpVfpmt5ujzHlEbKyJ6hgRAcWLmyDqjjLnCrbeaBFpICKvWa3mN5gwNQNUtY6d61U1HkiKnHEYEwf6oIhMFZHeVrZvgUsicgjYDIxX1UtZvB+H/MbJk9CpE7z2GqxaZc6JuxGYQ05gd/CyG/gCeB74LjuOD2xEzlBMaB1b4XUcCgBffglPPmkc+K1YYWyXHXKVrESfjMlVSRy8i3nzYPhwCAyE5cuhXj1PS1QoSFehReQxVV1qHfaTdLpJqvqf3BDMoYDTsyeMGwf/+EehcXKfH8iohR4CJCn0iHTyKCZGtIODcdS3YAF89hlUrgyzZnlaokJHugqtqt1c9h0zSYf0SUgwLfHUqcbM8eJFx0rKQ9id5Xb7LVhEduSsOA4FjvPnoVs3mDLFhJ7ZudNRZg9id1Ls1nTON8gpQRwKIKom5ExIiJkEGzrU+STlYTKLPjnf2i3msp9ELcw3ZYfCRmKi2Xx84P33jYVUs2aelsqBzFvo0+nsKyakrHcErnOwz6VL5ttyw4bGrW7r1p6WyMGFzCJnTAIzVlbVr/JGJId8y44d0K+fGTf36uVpaRzckNF36A6q+qN1eFVE3LqQUNXvc0Uyh/yDKrz7LowfD9Wrm+gVrVp5WioHN2TUQs/jz8mwxenkUSBjo2SHgk9YGLz0kmmVFyyAsmU9LZFDOmT0HfpWl/3q6eVz8GJOnjQB4WrXht27zTdmZxY7X5Mty3IR6QTEq+r2HJbHK4mLi+PUqVPExBSg5fBXr5oQNBcvgp+f8b7588+elqrQUaJECapVq4avr73IznZ9im0BJqnqDyIyDpgAxIvIbFV9M9vSFhJOnTqFv78/tWrVIr018fmGhATTMl+7Zlrm2rXB5svkkLOoKpcuXeLUqVPUrl3b1jV2LfybAkmt8V8wboLaAs9mVcjCSExMDOXLl8//yhwdbbxwhodD1apQv76jzB5ERChfvnyWenZ2u9xFgETL/ZCPqh60KiyXdTELJ/lemcE470tIMPGXb7rJ09I4kPX3xq5CbwNmA1Uwjg6wlNvxKFLQSUgw3eubbjKz1zfd5PjFLsDY7XIPAWKAI8Br1rlGwPu5IJNDLlC6dOm0J2NizETX0aPG7xdkqMxnzpyhb9++6aZHRETw73//23b+1AwZMoTatWsTGBhI8+bN2bhxo+1r84I5c+bwn//kc2thVc2TDeiO+UEIBSZmkK8v5vt268zKbNWqlaZHdMw1bbKwib48/+F08+QVhw4d8rQI6ufnl/LEpUuqQUGqwcGqERE5UseJEye0cePG2b5+8ODBunz5clVV3bRpk9arVy9H5IqLi8uRcjyFu/cH2KNudMLuLLcPJkzNIIzT/NPAZ8AMNW53M7s+01A4Vj5/jN+ynXbkKohMWXuQQ2eu5GiZjarcxGsPNLaXWRV++w0uXODk5csMmzaNi5cuUaFCBRYsWECNGjU4duwYjz/+OAkJCfTo0YO3336byMhIwsLCuP/++zlw4AAHDx5k6NChxMbGkpiYyMqVK5k0aRLHjh0jMDCQLl26MGrUqOT8CQkJTJgwgW+//RYRYcSIEYwePTpdMdu1a8fp03+aDwQFBTF27FgiIyMJCAhg4cKFVK5cmd27d/PUU0/h5+dHx44d+eabbzhw4AALFy7kq6++IiYmhmvXrrFp0yZmzZrFf//7X65fv85DDz3ElClTuHbtGv369ePUqVMkJCQwadIk+vfvz8SJE1mzZg0+Pj507dqVt956i8mTJ1O6dGnGjRtHSEgII0eOJCoqirp16zJ//nzKli1L586dadu2LZs3byYiIoJ58+bRqVPeuROw2+V+E+gFvIAJa/MC0AOYYfP65FA4qhoLJIXCSc0/gJmY7r1DbiBiutWVKvHcO+/w5JAh7Nu3j8cff5znn38egDFjxjBmzBh2795NlSpV3BYzZ84cxowZQ0hICHv27KFatWrMmDGDunXrEhISwqxU3krmzp3LiRMnCA4OTq4vI9atW8eDDz4ImO/4o0ePZsWKFQQFBTFs2DBefvllAIYOHcqcOXPYvn07RVMNF7Zv386nn37Kpk2bWL9+PUePHmXXrl2EhIQQFBTE999/z7p166hSpQp79+7lwIEDdO/enfDwcL744gsOHjzIvn37eOWVV9LI9+STT/Lmm2+yb98+mjZtypQpU5LT4uPj2bVrF7Nnz05xPk9w12yn3jBhbAJSnasAnLZ5fV/gE5fjQcAHqfK0AFZa+1tIp8uNCYOzB9hTo0aNdLspTpc7JX5+fqpXrpiDxERVVS1fvrzGxsaqqmpsbKyWL19eVVXLlSuX3E29fPlycnfdtUu9ePFibdSokc6YMUN/+eWXNOmpjx9++GFdv359hjIOHjxYa9WqpbVr11Y/Pz/dv3+/qqru379f/f39tXnz5tq8eXNt0qSJdunSRf/44w91fQf27t2bXN+CBQt0yJAhyWl//etftWbNmsll1K1bVz/55BM9cuSI1qpVS//2t7/p999/r6qmi96sWTMdNmyYrly5Uq9fv66qqq+99prOmjVLIyIitHr16sllh4aGaosWLVRV9a677tKtW7eqquq5c+e0bt26Gd6zHbLS5bbbQhcFElOdS8R9iBt3ZBgKR0SKAO8Af82sIFWdq6qtVbV1hQpOCJtMSUw0XezERDh3zpxL51NIVj6RDBw4kDVr1lCyZEm6devGpk2bMsyvqrbKnzVrFqGhoUybNo3BgwcnX9u4cWNCQkIICQlh//79rF+/PukHPl38/PxS1P/3v/89uYzQ0FCeeuopGjRoQFBQEE2bNuXvf/87U6dOxcfHh127dvHII4+watUqunfvbuM/8ifFixcHoGjRosTH522scbsKvQJYIyL3ikh9EbkP8/lqpc3rMwuF4w80AbaISBhwh1WfY2x7I1y/DkeOGHNHEahbN0Vy+/btWbZsGQCLFy+mY8eOANxxxx2sXGkebVJ6ao4fP06dOnV4/vnn6d27N/v27cPf35+rV6+6zd+1a1fmzJmT/IKHh4enK3aRIkUYM2YMiYmJfPvttzRs2JCLFy+yfbtZ2xQXF8fBgwcpW7Ys/v7+7NixI0NZAbp168b8+fOJjIwE4PTp01y4cIEzZ85QqlQpnnjiCcaNG8dPP/1EZGQkly9fpmfPnsyePZuQkJAUZd18882ULVuWH374AYDPPvuMu+66K9268xK736HHYz5XzQMqY5RxGWB3gJAcCgczoTYAEx8LAFW9jImTBSQvNR2nqntslu+QmuvXzaqvxESoU4eo6GiquUTrHDt2LO+99x7Dhg1j1qxZyZNiALNnz+aJJ57gn//8J7169eLmm29OU/znn3/OokWL8PX1pVKlSrz66quUK1eODh060KRJE3r06MGoUaOS8w8fPpxffvmFZs2a4evry4gRI3juuefSFV9EeOWVV5g5cybdunVjxYoVPP/881y+fJn4+HheeOEFGjduzLx58xgxYgR+fn507tzZraxgflAOHz5Mu3btAPMZb9GiRYSGhjJ+/HiKFCmCr68vH374IVevXqVPnz7ExMSgqrzzzjtpyvv000+TJ8Xq1KmT/L/zOO764bmxAT2BX4BjwMvWualAbzd5t+B8troxEhNVf/tNNTo6y5deu3ZNE61x9tKlS7V37945LV2OcfXq1eT96dOn6/PPP+9BaXKHHPtsJSL1Ma1yE+AnYJiq/prNH44MQ+GkOt85O3UUemJj4ddfjROC4sWhWrVsFRMUFMRzzz2HqlKmTBnmz0/tTi7/8NVXXzF9+nTi4+OpWbMmCxcu9LRIHiWzLvcHmC7yW5gu8mzg4dwWyiEbXLkCx4+bLnZ0tFHobKzLv4wAABnNSURBVNKpUyf27t2bg8LlHv3796d///6eFiPfkJlCtwKqq2q0iGwGHIPY/IaqCaJ+5owJOVO3LpQs6WmpHDxEZgpdTFWjAVT1qog4b0p+4/x5o8zlyhnvIo5hRaEmM4UuLiKu49ySqY5R1ak5L5ZDpiQmGi8iFSoYm+Vy5Rz3QA6ZKvR/gfouxytSHWf8Zd8h51E1rfKlS3DrraZFLl/e01I55BMyXFiiqoMy2Z7MK0EdgPh4CA2FU6eyHKJVRBg0aJBLUfFUqFCB+++/P9Nrk0wvw8LCWLJkSfL5PXv2JK//XrhwYYbflXOasLAwSpYsSWBgII0aNWLkyJEkJqZezJj7pDYZ9TR2V4o5eJpr1+DQITObXb061KmTpfGyn58fBw4cIDo6GoANGzZQtWrVLImQWqFbt27Ne++9l6Uysou7JZRJhiD79u3j0KFDrFq1ylZZqppjyp+RQickJORIHVkhW14/HW6AbybCuf1ZvEghKsp0t0uWhH2pFLlSU+iRueFbjx49+Oqrr+jbty9Lly7lscceS16+6GoaCNCkSRO+/PJLatWqlXz9xIkTOXz4MIGBgQwePJgWLVrw1ltv8eWXX6aoZ+3atUybNo3Y2FjKly/P4sWLqVChAg0bNmTbtm1UqFCBxMREGjRowI4dO1BVRo4cya+/miUOs2fPpkOHDkyePJkzZ84QFhZGQEBAih8TV3x8fGjfvj2hoaEAbs0kw8LC6NGjB3fffTfbt29n1apVHD58mJdeeomEhAQCAgLYuHEj165dY/To0ezfv5/4+HgmT55Mnz59WLhwIV988QXXr1/nxIkTDBw4kNdee42JEyemMBnt1asXU6ZMoXLlyoSEhHDo0CHefvvt5G/5w4cP54UXXkiWp2PHjmzbto2qVauyevVqSt7gFwqnhc7PqJoNMYpcyg+KZH8We8CAASxbtoyYmBj27dtH27Zts3T9jBkz6NSpEyEhIbz44ovp5uvYsSM7duwgODiYAQMGMHPmTIoUKcITTzzB4sUmZsN3331H8+bNCQgIYMyYMbz44ovs3r2blStXMnz48OSygoKCWL16dbrKDBAVFcXGjRtp2rRpumaSAEeOHOHJJ58kODiYUqVKMWLECFauXMnevXtZvnw5AK+//jr33HMPu3fvZvPmzYwfP55r164BsGvXLhYvXkxISAjLly9nz549bk1Gd+3axeuvv86hQ4cICgpiwYIF7Ny5kx07dvDxxx8THBwMwNGjRxk1ahQHDx6kTJkyyevnbwSnhc5rbLSkgGmRjx2DUqXSGFVkl2bNmhEWFsbSpUvp2bNnjpTpjlOnTtG/f3/Onj1LbGxssgvaYcOG0adPH1544QXmz5/P0KFDAaPchw796eviypUryUYevXv3TrfVSmoZRYQ+ffrQo0cPxo0bx/r162nRogUAkZGRHD16lBo1alCzZk3uuOMOAHbs2MGdd96ZLFu5csbf5fr161mzZg1vvfUWYDy2JvUcunTpQnlrAvLhhx9m69atyTbbrrRp0ya53K1bt/LQQw8lW349/PDD/PDDD/Tu3TvZ3RJAq1atCAsLy/L/OjW2FVpE7sYYVdyiqg+KSEvAX1X/d8NSOPyJKvz+u1nC6eMDFSvmaPG9e/dm3LhxbNmyhUuX/vTx6OPjk2JceSNBAUaPHs3YsWPp3bs3W7ZsYfLkyQBUr16dW265hU2bNrFz587k1joxMZHt27e7VVxXE8jUJLWMrqhlJvmXv/wlxfmwsLA05pTuzDlVlZUrV9KwYcMU53fu3Jkmf3rmoKnrSY/iLqv5ihYtmjy/cSPY6nKLyLOYNd2/AXdbp2OB129YAoc/SUiAEyeMo3t/fxN6xt8/R6sYNmwYr776Kk2bNk1xvlatWvz0008A/PTTT5w4cSLNtRmZR7py+fLl5Am3Tz/9NEXa8OHDeeKJJ+jXr1+yh5GuXbvywQcfJOdJraRZIT0zydS0a9eO//3vf8n3mWTO2a1bN95///1kRUzqHoOZSAwPDyc6OppVq1bRoUOHTP8nd955J6tWrSIqKopr167xxRdf5KpLIrtj6L8C96nqNP50dHAYuC1XpCqsJCSYEDRVquSak/tq1aoxZsyYNOcfeeQRwsPDCQwM5MMPP6RBgwZp8jRr1gwfHx+aN2/u1qQwicmTJ/Poo4/SqVMnAgICUqT17t2byMjI5O42wHvvvceePXto1qwZjRo1Ys6cOdm+v65duzJw4EDatWtH06ZN6du3r1uFq1ChAnPnzuXhhx+mefPmyevBJ02aRFxcHM2aNaNJkyZMmjQp+ZqOHTsyaNAgAgMDeeSRR2jdujXly5dPNhkdP358mnpatmzJkCFDaNOmDW3btmX48OHJw4FcwZ0JVuoNuAAUtfbDrb/FgbN2rs+tzWvMJyMikt0CaXx87gvkQXbv3q0dO3b0tBhZZsGCBTpq1CiP1J0bLoi2AuNSnRsFOOPnGyEx0YRqPXrUrPwCr16LPWPGDB555BGmT5/uaVG8FruTYqOBL0VkBOAvIgcxY+jcmyr1dmJizCx2dDRUrlwolm9OnDiRiRMnelqMbDFkyBCGDBniaTEyxZZCq+ppEWkFtMMEeP8N2K6qeb8UxhuIiDC2y0WKmLFyOm5zHByyiu2FJaqaqKo/qupSVd2aVWUWke4ickREQkUkzc+0iIwVkUMisk9ENopIzayUX6Dw8THfl2+7zVFmhxzFbuSME6RjWaWqdWxcbydyRjDGj1iUiDyDcbjvPa4orl+Hy5fNd+XSpaFhQ8fc0SHHsTuGHp7quDJmXL3U5vXJkTMARCQpckayQqvqZpf8O4AnbJad/4mKMoYVYCI8+vo6yuyQK9jqcqvqxlTbIuBBYLDNeqpixt1JnLLOpcdTwDfuEkTkaRHZIyJ7Ll68aLN6DxEXB3/7G1y8aHx8NWrksQDq7qJP2ommmJFZ5BtvvJHudZGRkTzzzDPUrVuXFi1a0KpVKz7++GMgpelj8+bNad++PUeOHAFgy5YtiAjz5s1LLis4OBgRSV6O6ZA+N2KcEQ1k2t22yDByRoqMIk8ArYFZ7tK1oETOUIVevWDWLNPFvvXWG3LclxuMHDmSJ5/Mvkl7Rgo9fPhwypYty9GjRwkODmbdunUpnOsnLdvcu3cvgwcPTlFW06ZN+fzzz5OPly1bRvPmzbMtZ2HC7hg6tbvdUpjgdett1pNZ5Iykeu4DXgbuUtXrNsvOn4jAoEEwdKj5JFXE/Ha+uetNfg7PWV+Lt5a7lQltJmT5OleTyfSiOIKJ89y9e3eOHTvGQw89xMyZM5k4cSLR0dEEBgbSuHHj5HXZYIwmdu3axZIlSyhi3XeFChWYMMG9jFeuXKFs2bLJxzVq1ODKlSucP3+eihUrsm7dulw1JvEm7I6h66c6voaZ5Fpo8/oMI2cAiEgL4COgu6qmXXxbEEhIgClToF49ePJJo9BgIljkc4YOHcrcuXNp3759mm/FISEhBAcHU7x4cRo2bMjo0aOZMWMGH3zwgdt11wcPHqR58+bJyuyOJEupq1evEhUVxc6dKSMI9+3bl+XLl9OiRQtatmyZwpDBIX0yVWhrhnoD8F9VzZYJjqrGi8hzwLeYwHfzVfWgiEzFLGFbg+lilwaWW1Ysv6pq7+zU5xHOnYOBA2HzZhg1yii0G7LTkuY2ERERXL16lfbt2wMmEJ2r04J77703OcRMo0aNOHnyJNWrV3dbljtef/11li9fnhxLClJaSn3++ec8/fTTrFu3Lvmafv360b9/f37++Wcee+wxtm3bdsP3WRjIdAxtfW9+P7vK7FLO16raQFXrqurr1rlXLWVGVe9T1VtUNdDaCo4yb9kCLVrAjh2wYAG4WA4VBDSTKI6pzfwyi6jYqFEj9u7dm2yO+fLLLxMSEsKVK+4D3ffu3TvZCUESlSpVwtfXlw0bNnDvvffauQ0H7E+KfSUiziDGHb/8AvfdZxaI7NoFBWB5YGqyEsXRFV9fX+Li4tKcr1evHq1bt+aVV15J9quVFPjNHVu3bqWuGycOU6dO5c0330wTyN0hfeyOoYsA/yciWzGfn5KfjKoOyw3B8j3x8WbFV4MGsHAh9OmT47bLOUlUVBTVXGJdjR07NkW63SiOrjz99NM0a9aMli1bppgUA/jkk08YP3489erVo1y5cpQsWZI333wzOT1pDK2qFCtWjE8++SRN+UlDAAf7SGbdLQAR+Ud6aao6Kb203KZ169a6Z4/7iLMx16O4fVlb+iQ2YNrQG/fVlIJt28wYeckSaNMm0+yHDx/mttvyt+l4ZGRk8rfqGTNmcPbsWd59910PS+UA7t8fEQlS1TTx0zOLPvmYtXbbY0qbr1CFd96BCROgRg0oVszTEuUYThRH7yCzLvdH2F/e6d1ERJjx8erV8NBDZvLLiwwrnCiO3kFmk2LOguMkPv4YvvrKtNArV3qVMjt4D5m10EUtb5/pKraqbspZkfIRqiayY9WqMHYsdOsGzZp5WioHh3TJNPokxttnegqt2F/PXbC4ehVGjDDfmA8cgIAAR5kd8j2ZKfQ1O/bOXse+ffDooyYw3Ouvm1CtDg4FACcUjiuqMH8+tG1rWuhNm2DixGTDioLOuXPnGDBgAHXr1qVRo0b07NmTuXPn2opA6VAwcCbFUrN6NXToAMHBcNddnpYmx1BVHnroITp37syxY8c4dOgQb7zxBufPn/e0aA45SIZdblXNv0ufcpKffzbxlmvVgsWLTWC43Fxu2Llz2nP9+sGzzxrvJu5MBYcMMdvvv0PfvinTtmzJtMrNmzfj6+vLyJEjk88FBgYSERHBxo0b6du3LwcOHKBVq1YsWrQIEWHq1KmsXbuW6Oho2rdvz0cffYSI0LlzZ9q2bcvmzZuJiIhg3rx5dOrUiYSEBCZMmMC3336LiDBixAhGjx5NUFAQY8eOJTIykoCAABYuXEjlypWz8h9zsIl39CVvhCVLoHVrYyEFxhmBF64dTlJWdwQHBzN79mwOHTrE8ePH+fHHHwF47rnn2L17d3JcaVcLrPj4eHbt2sXs2bOZMmUKAHPnzuXEiRMEBwezb98+Hn/8ceLi4hg9ejQrVqwgKCiIYcOG8fLLL+f+DRdSCm/0yZgYeOEF+Ogj6NgR5s7Nu7ozalFLlco4PSDAVoucFdq0aZO8zjswMJCwsDA6duzI5s2bmTlzJlFRUYSHh9O4cWMeeOABwERRhJRRE7/77jtGjhyJj495rcqVK8eBAwc4cOAAXbp0AUwQdKd1zj0Kp0KfOgUPPAAhIWYZ57RpxtDCi2ncuDErVqxwm+bOPDImJoZnn32WPXv2UL16dSZPnpwiImXSNa7mlOomoqOq0rhxY7Zv357Tt+TghsLZ5S5TBvz8YO1amDHD65UZ4J577uH69evJjvoAdu/ezf/+5z6aUZLyBgQEEBkZme6PgStdu3Zlzpw5yQoeHh5Ow4YNuXjxYrJCx8XFcfDgwRu9HYd0KDwKHRsL06fDtWtmnPzDD1CIPteICF988QUbNmygbt26NG7cmMmTJ1OlShW3+cuUKcOIESNo2rQpDz74ILfffnumdQwfPpwaNWrQrFkzmjdvzpIlSyhWrBgrVqxgwoQJNG/enMDAQMf7SG7iLoJdbmxAd+AIEApMdJNeHPjcSt8J1MqsTNvRJ8PCVNu0UQXVxYvTvSa3yDT6pINDBuRG9MkbwiVyRg+gEfCYiDRKle0p4A9VrQe8A7xJDtAw5KxxD/Tzz7BihfH75eDgpeRVlzs5coaqxgJJkTNc6QN8au2vAO6V1DMsWaTfpksMem871KwJQUHwyCM3UpyDQ74nrxTaTuSM5DyqGg9cBtLEWM1K5IztjUuzrUtd42GkXr0bkf+GURueYRwcUpPV9yavFNpO5Axb0TXUZuSMEsVL8fXffqH9+lCz8suDlChRgkuXLjlK7ZAlVJVLly5RokQJ29fk1fcaO5EzkvKcEhEf4GYgHC+gWrVqnDp1inwfi8sh31GiRIkUzh0zI68UOtPIGcAaTPC77UBfYJN6SZPm6+tL7dq1PS2GQyEgTxRa7UXOmAd8JiKhmJZ5QF7I5uDgTeTZEilV/Rr4OtW5V132Y4BH80oeBwdvpPCsFHNwKATYcrSfXxGRi8DJTLIFAL/ngTiZkV/kAEcWd+QXOcCeLDVVNc1nngKt0HYQkT3qJsJAYZUDHFnysxxwY7I4XW4HBy/CUWgHBy+iMCh0HroiyZD8Igc4srgjv8gBNyCL14+hHRwKE4WhhXZwKDQ4Cu3g4EV4hUKLSHcROSIioSIy0U16cRH53ErfKSK1PCjLWBE5JCL7RGSjiNT0lCwu+fqKiIpIrny2sSOHiPSz/i8HRWRJbshhRxYRqSEim0Uk2HpGbpyk54gc80XkgogcSCddROQ9S859ItLSVsHu3JgUpA2zNvwYJmheMWAv0ChVnmeBOdb+AOBzD8pyN1DK2n/Gk7JY+fyB74EdQGsP/U/qA8FAWeu4ogefz1zgGWu/ERCWS7LcCbQEDqST3hP4BmNWfAew00653tBCe8QbSnZlUdXNqhplHe7AmJLmBnb+LwD/AGYCMW7S8kqOEcC/VPUPAFW94EFZFLjJ2r+ZtGa+OYKqfk/G5sF9gP+oYQdQRkQydWjuDQqdY95Q8kgWV57C/ArnBpnKIiItgOqq+iW5h53/SQOggYj8KCI7RKS7B2WZDDwhIqcwxkSjc0mWzMjquwR4h6P9HPOGkkeymIwiTwCtgdyKiJehLCJSBOOMcUgu1W9LDgsfTLe7M6bH8oOINFHVCA/I8hiwUFX/KSLtMCa9TVQ1MYdlyYxsvbPe0EJnxRsKuewNxY4siMh9wMtAb1W9ngty2JHFH2gCbBGRMMw4bU0uTIzZfT6rVTVOVU9g3D3Xz2E57MryFPBfAFXdDpTAGEvkNbbepTTkxoA/LzfMr/txoDZ/TnQ0TpVnFCknxf7rQVlaYCZm6nv6/5Iq/xZyZ1LMzv+kO/CptR+A6WqW95As3wBDrP3bLCWSXHpGtUh/UqwXKSfFdtkqMzdfqrzaMDOCv1iK8rJ1biqmBQTzK7sc48R/F1DHg7J8B5wHQqxtjadkSZU3VxTa5v9EgLeBQ8B+YIAHn08j4EdL2UOArrkkx1LgLBCHaY2fAkYCI13+J/+y5Nxv99k4Sz8dHLwIbxhDOzg4WDgK7eDgRTgK7eDgRTgK7eDgRTgK7eDgRTgKnc8QkUUiMtnTcmSGZbHUKYP09SLyeF7K5OAodK4hImEiEi0ikS5bFQ/JskhEYi0Zwi1la3AjZapqQ1X9wSp/mogsTJXeVVUX30gdqRERH8vM85p1L6dEZJa1jNXO9fdZq+K8Fkehc5cHVLW0y5Yrljs2eUNVS2OWE4YD8z0oy43S2LqXe4BBmJhoDjgKneeISBERWSEi50QkQkS2iMht6eStKCJfW/nCReR7l7RqIvKFiFwUkRMiMspO/ap6DbNKqYlVTgnLkP6siJwWkbdFpJiN+k+JSGcRuR/4G/C41WoGWelbRWSIiJQUkSsicqvLtZWs3kt567i3iOy16tkqIk1s3ssvwDYg0KXs4SJyWESuisgxERlunb8ZWAvUcOkxVbSex0tW3t9FZJmIlLVTf37EUWjP8CXG+KAScAD4LJ184zFrjytYeScBiEhRq4zdGJO6LsB4Ebk3s4pFxB8T+TPYOvUqxuqrGWadeQfg7xnV74oa08uZwGKrF9IqVXo0sApjxZREf2Cjql4SkduBj4HhGJPW+cDqpB+VTO7lNkveUJfT5zHroG/C2Fm/LyLNVPUy8ADwq0uP6QIw1sp/J8YA4hrwXmZ151tya81sYd+AMCASiLC2VenkC8CYxflZx4uAydb+G8D/AXVTXdMBOJ7q3CTg43TqWIRxYBCBWT+8CqhtpZ3EZb0y5uUOzah+K+0U0Nnan4YxOXRN38qfRg7dgV9c0nYCA639j4HXUl17DOjgpk4f6391BaN4at1bsQyew5fAKGv/PlJ5IAGOAne5HFcHrgNFPP0OZWdzWujc5UFVLWNtD4JpXUVkpogcF5Er/Nm6uDPRm4FRuI1Wl3C8db4mpusYkbRhur2VMpBlhiVHZVV9UI2ZIkBlUsYHO8mfhvTp1Z9VvsN43GglInWBxsBql3uZkOpeKpOxMX8zjPnnQKAdUCopQUTuF+M3LtwqqysZmz/WANa61L0f80NRMVt36mEchc57nsRY/NyDscuuZ51PY9CuqldU9UVVrQU8iHnx78KYFx51+bEoo6r+qvpANuQ5i1GqJGoApzOpP42oGVWgxkvMcky3eyDG9vmalfwbMCXVvZRS1f9mUmaiqi4F9gCvAIhISYyLqenALapaBljPn/9bd3KeArqkqr+Eqp7LqP78iqPQeY8/pkt3CdOyvJ5eRhF5QETqiohg3CYlWNt2IFZE/mpNahUVkaYi0iq9sjJgKfCqiASISAVM131RJvWn5jxQy8qXHkswY+eB1n4Sc4FRInK7GEpb9frZlH86MNKSvTjGzvkikGBN2LnOK5wHAqx5hCTmAG+ISA3rniuKSG+bdec7HIXOexZgjObPAAcxs7Tp0RDYhBmL/wi8q6pbrRavJ8bpXRgm9OhH/OncLitMwdj+7gf2Yca30zOq300Zn2MUKVxEdqVTzzYgHjPBtj7ppKruxHg//RD4A2Or/IRd4VU1BPMDN06Ny6IXgS8wn+b6YsbQSXkPACuBMKuLXRFjh70OM6y4asl5u9368xuOPbSDgxfhtNAODl6Eo9AODl6Eo9AODl6Eo9AODl6Eo9AODl6Eo9AODl6Eo9AODl6Eo9AODl7E/wOMRL2scSnCJwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 252x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(3.5, 3))\n",
    "for idx, label in enumerate(['Logistic Regression', 'Multilayer Perceptron', 'Light GBM']):\n",
    "    fpr = mdl_res[idx]['multiclass_auc']['fpr_A']\n",
    "    tpr = mdl_res[idx]['multiclass_auc']['tpr_A']\n",
    "#     plt.plot(fpr, tpr, label=f'{label} (auc={metrics.auc(fpr, tpr):.3f})')\n",
    "    plt.plot(fpr, tpr, label=label)\n",
    "\n",
    "plt.plot([0, 1], [0, 1], 'r--', label='Chance')\n",
    "plt.xlabel('False Positive Rate', size=12)\n",
    "plt.ylabel('True Positive Rate', size=12)\n",
    "plt.title('ROC for Severe vs. Rest', size=12)\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.savefig('../archive/Model_Group_ROC.png', dpi=600)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-06T17:43:12.442874Z",
     "start_time": "2020-02-06T17:43:11.904125Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPQAAADQCAYAAAAwEkzeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOydd3wVVfbAvyd56QmEFDohFAGpQVmUomJDsYCFBeyI4GJBlJWFXRu6FiyrrOuuiIqoiAX4iWBFKSpKl4D0IkGK1JDe887vjzsJj/CSvIS8vCTM9/OZz5ty594z8+bMLXPPOaKq2NjY1A38fC2AjY1N1WErtI1NHcJWaBubOoSt0DY2dQhboW1s6hC2QtvY1CFshT5NRKSPiOwQkQwRua4ay71FRBZWV3lVQS2VeYaIPG2tXyAi2yqZz1QReaxqpXODqtaJBUgCsoEM4CAwAwgvkaY3sBhIB1KBBUDHEmnqAVOA3628dlrbMaWUuwgYW4XXMQNQYGCJ/VOs/cO9cO+WAiN9/R9WUvbhQKH1X6UBicA1Vfx/PF0JmZb54n7UtRr6WlUNBxKA7sDfiw6ISC9gIfAZ0BRoBawHfhKR1laaQIyCdgKuxCh3b+AY0LOUMlsCmyojrIg4Sjm0HbijRLo/A7sqU84ZwHLrf48E3gY+EZGokonKuN91B1+/YavwTZoEXOay/QLwhcv2j8D/3Jz3FfCetT4SOESJmr2MMncBTk60DIIwL4v5QDKmdh/lkn4SMAeYialNTqkVMTXCS5hWRgNr3zWWnMuwamhK1AKY2ns0sAM4DvwXEJdyZ7qkjbfSO4BnMDVcjnUNr1lpOgDfWtexDRhSxn34HrjRWu9r5X2VtX0ZkFhSZkCAV4DDmNbSBqCzdSzIuge/W//HVCCklLJL3ocwq/weQD9gHzDBup/vu9zPRCAF+Bno6nJ+d+AXTCvuY+AjrBq6KD+XtC2A/wOOYF76rwFnW/eyqNWQ4vK/Pu1y7ijr+UjGPC9NPfkvz7QaGgARaQ4MwNwwRCQUU9POdpP8E+Bya/0y4GtVzfCkHFVtg3norlXVcFXNBT7EPERNgcHAsyJyqctpgzBKHQl8UErWOZg/eZi1fTvwngciXQP8CegGDAGu8OAaHsG87O63ruF+EQnDKPMsoCFwE/A/EelUSjbfYx52gAuB34CLXLa/d3NOf+tYO8y9GIpRCoDnrf0JQFugGfB4eddi1cAjMYq0w9rdGIjCtKTuFpFzgOnAX4Bo4A1gvogEWS20ecD71jmzgRtLKcsf+BzYg3lBNgM+UtUtGGVcbt3PSDfnXgI8h/mPmlh5fFQiWYX/S6h7g2LzRCQd2It58z9h7Y/CXOsfbs75A4ix1qNLSeMRItICU0NNUNUcVU0E3gJuc0m2XFXnqapTVbPLyO494HYRqY9RjnkeiDBZVVNU9XdgCUYhKsM1QJKqvqOqBar6CzAX84Jyx/ecrMDPuWxfhHuFzgciMC0BUdUtqvqHiAim9npIVZNVNR14lhMvN3ecLyIpmFr4JuB6VU21jjmBJ1Q117rfo4A3VHWlqhaq6rtALnC+tQQAU1Q1X1XnAKtLKbMn5qU9XlUzrf97WRkyunILMF1Vf7Eqgb8DvUQk3iVNpf7LuqbQ16lqBKa26MAJRT2O+WObuDmnCXDUWj9WShpPaQoUPYRF7MG8vYvY60lG1sMRCzwKfF6O8hdx0GU9Cwj3pCw3tATOE5GUogXzEDYWkThrRD9DRIpaMsuBdiLSCPPgvQe0EJEYzIP/g5vrW4xpov4XOCQi00SkHuaaQ4G1LmV/be0vjRWqGqmqMap6vqp+53LsiKrmlLi2v5a4thaY/64psF+tdq/FnlLKbAHsUdWCMuQqjaau+VotwmOc/JxU6r+sawoNgKp+z4m+KKqaiXno/uwm+RDMQBjAd8AVVpOzMhwAokQkwmVfHLDfVbwK5DcT+CueNbfLIhOjJEU0LnG8pEx7ge8tJSlawlX1HlX93VoPVzMQhapmAWuBscBGVc3D9E3HAbtU9ShuUNVXVfVczCBkO2A85uWaDXRyKbt+UVmVwN21PVPi2kJV9UNM66yZ1UooIq6UfPcCcaUMtJX3Hx/AvFgAsJ63aE5+TipFnVRoiynA5SJS1FSZCNwhIg+ISISINLC+L/YCnrTSvI/5o+aKSAcR8RORaBH5h4hcVV6BqroX8yA/JyLBItIVuIvS+8rl8Sqmf39KDVdBEoELrdq1Pi6j/xaHgNYu259jatzbRCTAWv4kImeXUcb3wP2caF4vLbF9ElZ+54lIAOaFkwMUqqoTeBN4RUQaWmmbiYhHfUgPeBMYbZUtIhImIldbL+HlQAHwgIg4ROQGSv+6sQrzAphs5REsIn2sY4eA5laf3B2zgDtFJEFEgjBdipWqmnS6F1dnFVpVj2Bqtses7WWYgYUbMH/EHsyIZl9V3WGlycUMjG3FDAqlYf64GGClh0XfhBkkOQB8ium/fVvJa0hW1UUlmoCVyedbzIjtBkxN+nmJJP8GBovIcRF51eoy9Mf0Ww9gmn/PY0afS+N7TJ/4h1K2S1IPo1zHMf/FMawWFWZUeiewQkTSMC2n9h5dbDmo6hpMP/o1q+ydmJFyrJbFDdb2ccxA3f+Vkk8hcC1m0O53zEDoUOvwYsynzIMickrrRFUXYZ7LuZhnsQ1ljxF4jJzms2JjY1ODqLM1tI3NmYit0DY2dQhboW1s6hC2QtvY1CFq9WT1mJgYjY+P97UYNjbVztq1a4+q6imTbWq1QsfHx7NmzRpfi2FjU+2IiNsZbHaT28amDmErtI1NHaJaFFpEpovIYRHZWMpxEZFXRWSniGywTNxsbGwqSHXV0DMwHkBKYwBwlrXcDbxeDTLZ2NQ5qmVQTFV/KGHrWZJBGK8hipm/GykiTVS10rbJNjanjSoU5oMz3/zmZ0FeFjgLoDAPCnKsfZmQn22WvAzIzUBz00jLSeFYbgp5zlwyCnPZ4Mzid82j0OlEnU6cTidZOMlWJ05V/LLz6bIumd19OvGvkV9XSuSaMsrdjJPthPdZ+05RaBG5G1OLExdXmmWbzRmLKuSmQ+YRyDwK2cchJxXy0iE/xyhiXoZJk59l9uVnW+tZkJsBuWmQnWLOKUEBcNTfn98CA/glKIg9AQ4OO/xJ9fMj3c+PbPEj208oOMkC8wT1ChU/NccUCFIIcgot9+fwyBu/0/RwLq+2aVfpy68pCu3u6t1ajajqNGAaQI8ePWzLkrpKXiakHzS/hXnmNzcNso6dUNacVMhJg6yj1naK2ecs2+eAij8EReB0hOD0D6LAP4R8vyByJZhcv0bkhLQiJTSUnTjZoVns0QwOSgbHHFlkOLJOPK0qBOSH41cQDnmhOJ0h5DtDKXAGgwQS5BdGiF8k4UEh1AsKoVVEO1pGNqJR/WCiQgOJCHYQGRpIZGgADX5aSuBbd8GimTx48cWVvm01RaH3YTxAFNEcY7ZnUxcpyIOU3yF1r1HatP1m++h2SNlrFDOvHLduQfUhuD4aFE5hSAw5UZ3I8AsnU8LJ9IsgxS+S49Qn2RnKwbwQ9mb5cSATjmQ6OZwj5GcDFCABKfgFJuMXcNysB6TgF3AMv6AtiF++KUv9CfNrTKSjLW2CmhEb0ogmYc1pW68j9YLCCXT4ERbksBQ0gPohAQQ5/Mu/D1lZsHgxXHMNDLgSdu6EkJDTurU1RaHnA/eLyEfAeUCq3X+uxahCxmFI3WeUNnUvHN8Dx5Pg2A6jvOo8+ZzQGIg5i/y4PmT6RZDmH0mKfwxZBJPtdJCcH8DBXAe/ZQazJzeMlDw/UjLzOH4kn0JnKQ01ySc4JJWI8IOEh6cRGJ1KeKMUAuUYWc4jZBWm4NoQ9Bd/YkIa0iS0KZ1iLqRTTEc6RHUgvn48AX4BVXuPtm2DwYNh61bYsQPi409bmaGaFFpEPsT4+YoRkX0Y530BAKo6FfgSuApjbJ4F3FkdctlUAapwdAfs/A6OboOjO+HQr6bp60pQPYhsibNJdzLOuo5jQS34vSCarVlhbMkMZ3eKkz37skjJyi+1qHrBDpo3CCUqPIDWkXkEBRfgCMzAEZAF/ukUSirZzuOk5R/jUPZ+DmcdQlFyMO5QAvwCaBTciDbhzWga3okmYU1oEt6EFhEtaBbejNiQWPz9PKhZT5ePPoJRoyA4GD7/3ChzFVGrHRz06NFD7amf1YgqHN8NB9bBwV/h4EY4tBHSrcZUaDREtYZGncmN7sCegig2Z0bwS2oEW1P82Xc8i4NpObhWqAH+QvMGoTSNDKZJlBIZnkVoSA5+jgxy9DgZBcmk5yeTVZhGam4KqbmpJOckk1OYc4p4gX6BxIbGEhsSS/OI5sRFxNE8ojnNI5rTIqIF0cHRSCmDVdXG2LHw6qvQp49R7ObNK5WNiKxV1R4l99eUJrdNTUQVDm+BHQthz8+wb5UZNQbwC0Bj25PZtA/7wjuzPrgnv2bWY9fhTPZszORAapHC5RFd/wBNo/Jp1TKL9sHHEUca6p9JAelkFhwnPT+NHbmpbEjPNu7tXQj2DyY2NJYGQQ2IDommTWQbooKjaBjakNiQWGJDY4kOjqZBcAMigyJ9r7DlER8P48fDM89AQBU347FraBtXihT4tyXw2/ewf40ZVQay6rflj4gubA9oz7r8eFZnNWZncjqZzqNIwHH8AlIIDs6gXng2QUE5iCOdHD1KWn7yKcWEBYQRGRRJg6AGxITEUD+oPvWD6tMotBENwxoSHRxNdHA0MaExRARE1HwlLY9584zyXn11lWVp19A27kn5HXYtNkvSsmIFPhIUxwbpziKJ49vCliRn5uNfcBhH4C6CQ9dCvUNQPwVXf8d+4kdAUAPqBzcgOjiaZhGdaRzamNjQWBqGNqRhaEOahjelXmA931xrdZOfDxMnwssvw+WXw1VXgZdfTrZCn2lkHoOd31G4+0cKdi/DkbqbAw4HWwKj+CGwJWuD2vOHw5+CwCz8Avagsh044dS7QVADmoU3o3XkhbSq34pm4c1oEtaExmGNiQmJweFnP1IA7N0LQ4fC8uUwZgy8+KLXlRlsha775GXCwY0UJv3Eoc2fc/j4Jg46/FgdFMbK0HD2RcZT6Ff0CekoQRJOi7AWnBXVnkZhDYtr1mbhzWgT2YaIwIgyi7MB9u2D7t0hLw8+/hiGDKm2om2Frms4nejhLWza9Cmbdy7kYPYetgQFsCUwkGMh/hDSEAB/AogLO5urG3binMYdiKvXgpb1WhIbElv7+6y+plkzUyvfdBO0q/w0zspQYYUWkYaqetgbwthUkvwcjvy6gB82fMjGjE0sD/Zjf4ADgkGC6lG/sAExwe3o2bArvePO5uzYOFrXb02Af9WPsp6x/PEHjBxpmtYdO8ITT5R/jhfwSKGt8Cn/wcSBKgTCRORaoIeq+kbyMxxnyu8sW/EGy37/ns16hE1BARSIEBAWRHNnY64Iv4g+7ftz2VmdiQg6/RlINmWweLGpjTMyzKyvjh19JoqnNfTrmPhDZ2HCqYAJDfMvToRstfEySUc3883yqWw/8BPrHFkccTiQQKVZfn3OD+jCee1vZHDXiwkPtBW4WigsNN+TJ02CDh2MYncqLYR29eCpQl8GNFfVPBExIeZVD1vhQ228yLZD65i/eipLDq9mr7+ZFhkRoDQtaEKfqAEMO/82OjUpGUzSplqYOtU0rW+9FV5/HcIrGyCz6vBUodMwQdOLY9Zawc0PeUOoMx0tyGfdhhn879cZrCQNP1U65znpXtCSTi1v5LILb6FhfbsW9hm5uRAUZPrMDRsaI4saMpDoqUJPB2aLyD8APxH5E/Ac8IbXJDvTUCX/9+XM/vkVZmdsYmegP/ULnVyS2pD2jW5l4MChNI/yfQ1wRuN0wksvwVtvwcqV0KAB/NldyHHf4alCPwfkAW8DwZj4tm8Ar3hJrjMHVfK2zOfj75/h3cB0DjkcNNYg+mT/iX4JDzDonLMJCawGCyCbsklOhuHDYcECUyP718z/xFOFjlbVlzgRvxcAEYkBTol/a+MZuzavYcXih/g46Ai7QwOIyYnhmthRPNRvCA3rBftaPJsiVq0yk0MOHDCWUvffX2Oa2CXxVKF/wwToLsl2TN/axkPU6WTryq9YveIlvgzdz6aIIKIKG3BL3Fge6vtnggJq5pv/jOaxx8zvsmXQs6dvZSkHTxX6lNeRiIQDTjdpbdygqqz//jP2r5zEnPqZrIkKJkrrM7HLvQxLuL16DOttPCc11UzdjI2F994z1lJRNb/uKlOhRWQ3xkdLiIj8VuJwDDDXW4LVJTZv38HyBQ+wPGwnKxsHU48GPJwwmps630qgf6CvxbMpSWKiGexq0wa+/hoa1Z6vs+XV0CMxtfN8YJTLfgUOqeomTwsSkSuBfwP+wFuqOrnE8TjgXSDSSjNRVb/0NP+ayOakA6z58nnW6Zd8FxNMfeoxtttfuLnTrYQGhJafgU31ompGsMeMgZiYE03tWkSZCq2qiwBEpLGqplW2EBHxB/4LXI7x8LlaROar6maXZI8Cn6jq6yLSEeNnLL6yZfqStPQ0Fr77BHvz5/NhZDAFEszI1oP5S68JBDvswa4aSWYm3HMPvP8+9O8PM2ea5nYtw6M+tKqmiUhn4AJMU1tcjj3lQRY9gZ2q+huA5d1zEOCq0MqJgbf61FI3vr+vXsDRhQ/xQUN/dkaEcnH0OTx84T+Jq2cHBajR5OSYQa+nnoJ//KPGfpYqD0+NM+7CGGcswtSy3wKXAgs8LMddZIzzSqSZBCwUkTFAGGa6qTtZambkjIJc9s+eQOL+T3i2SRQSEMbUS16mT7M+vpbMpiwWLIArroDoaNi4EUJrd1fI02B1E4GrVPVaINv6HYIx2PAETyJj3ATMUNXmGJe+74vIKfKp6jRV7aGqPWJrSpPo8Fa2/qcPk1I/55HYaOKiOzP7urm2MtdksrONK92BA+ENa8JjLVdm8PyzVSNVXWqtOy1F+wJ4DxjhwfmeRMa4CytCpaouF5FgTPO+Rtte565+j6++f5TnoyLI8Ytg3DkPc0fnm/A79V1kU1PYscPM9tqwwTSv77nH1xJVGZ4q9D4Raamqe4AdwNWYGWKle0U/mdXAWSLSCtgPDANuLpHmd0wzfoaInI2ZYnrEw/yrn/SDbP30fl7KWs/K2Po0crTmw2v+Q3z9GtQNsDmVL7+EYcPMd+Uvv4QBA3wtUZXiqUL/C+gM7AGeBmZjIl+M8+RkVS0QkfuBbzCfpKar6iYReQpYo6rzgb8Cb4rIQ5jm+HCtiT6GVSFxFl8ufYynIkPIDwpjWPx9TLxghD05pDbQooWZ7TV9OtSkMZgqolJ+ua3mcJCqppab2ItUu1/u3Azy547kpaMrmFU/guDcprx25X85L65t9clgU3F27zbO+iZO9LUkVUZpfrkr1dFT1RzAISLPnbZktYWsZFLeH8hd6b8wq34EARkXMeu6T2xlrunMnw/nnAOTJxvXunWcchVaRO4QkVdE5F4RcYhIPRF5EUgCzvG6hDWBnDQOvH8tt/EHiYEhRKQNZ8EtL3JWw/q+lsymNPLzTciZQYOgdWv45RfT3K7jlDeX+wXgNuBnzGel84FewFqgr6qu97qEvqYgl20f/5l7HMc57heC4+jdfPqX4TSyzRtrNtddZwa97r0X/vUvE+nxDKC8QbFhwIWqusMaed4E3KSqH3tftBpAYT5rPhnC/c59+PlHkJZ0N+/eMshW5trAqFFw221mRPsMojyFjlTVHQCqukVEss4kZd718TAeyNlJPUckO7bfx5iLetD3rBhfS2bjjsJC432zUSPjgOC663wtkU8oT6HFcgZYNNOroMQ2qvq7t4TzJUe/nsi9WZtxBEWQtPM+LmzdlrGXnuVrsWzccfAg3HwzLFkCo0f7WhqfUp5Ch2EGv1ynbu5xWVfMd+U6RcGm/2P83vkkh4QSljyGmOBGTBmagL9fzXQ7c0azdKlxcp+aCu+8Y/x+ncGUp9BnXqyUozv595IJrIkI5uL69zJ/SxTvDO9CgzDbEUGNY88eE6a1TRv49lvo3NnXEvmc8uyhC6tLkBpBXiZfzR3GjIhgBjS+gnk/xHFtt8Zc3KGhryWzcSUvDwIDoWVL+OgjY78cYUfFhEpOLKmrbF1wH48HZpMQ0Yp1mwdQL8TB49f4Lk6RjRt+/tlEdFyyxGzfeKOtzC7YCm2RsvNbHjy+knqOUBrq39hxKId/DUkgNiLI16LZgJlD//LLcNFFxvlAfXtSjztshQYK83OZ+MPfOOzwZ0SHZ5m7OpW7L2zNRe1qiL31mc7x43D99fDXv8K118LatWY6p80peKzQ1rTPXiIy2NoOEZE6EWBp5qKH+Mm/gL/FXcu0pQ5axYQx7vLqDdRtUwYffwxffAGvvAJz50JkpK8lqrF4pNAi0gnYCrwPzLB2X4qJeVWryc9O4b0DP9BTg9ibezt7jmXxzHWdCbYd3vsWVWMlBfCXv8D69fDggzU2YkVNwdMa+nXgaVVtywmnBksxTgNrNYsW/53D/sL1He7i7Z+TGJTQlN5t7dlgPiUtzUzZPOccE35GxKdB1GsTnip0F4zPbLB8galqBlC7nTBlHmXm/iW0kEDWH7uQvAInD15mN7V9yoYN0KOHaVpPnAiN7djXFcFThd4DdHfdISI9gF1VLlE1smnFFNYHBXB966HMXLGXQQnNaBUT5muxzkxUjReR886DjAxYvBgmTAA/e9y2Inh6tx4HvhCRx4BAERkPzLH2e4SIXCki20Rkp4i4dR0hIkNEZLOIbBKRWZ7mXSlUmbP7C4IVDqdfSk5BIfdfYjsr8ClLl0KfPrBuHVx4oa+lqZV46mh/voj8gQmH8xPQHhiiqqs8Od+TyBkichbwd6CPqh4XEa9Oz8rau5yv/PO4vH4n5q5J5vKzG9Em1g6oXu1s2WL6yB06GHe6gYG11sl9TcBTR/sNVHU1xntnZfAkcsYo4L+qehxAVb3qvnfh6lfJ9POjYb0hJGfmMbx3vDeLs3HHrFlw993wpz+ZmV8hdeIrqE/xtMm9X0Tmi8jQSn57dhc5o1mJNO2AdiLyk4issILbnYKI3C0ia0RkzZEjlfTyW5DHp8nraSnBLNzUkHaNwunVJrpyedlUnJwcY+Z4yy3QvbuJI2VTJXiq0K2A74CHgEMi8r6IDLCa0p7gSeQMB3AW0A/j7ugtETllBkFVRM44uG0BvwQ6uKBBLzYfSOf2XvGI/X2zevjjD+jd2zSvJ0wwNXOzku92m8rikUKr6iFVfVVVzwcSgG3AS3geUM6TyBn7gM9UNV9Vd1tleMWjwLKtnwBwJPdSghx+DEpo6o1ibNwRFWWiOi5YYDxxOjx1DW/jCZX5JlDfWiLwPLZVceQMEQnE+CqbXyLNPOBiABGJwTTBSwaZP31U+TF5E00IYMnWEC49uyERwWee2Xe1kpdnojqmpEBQkAmifs01vpaqTuLp1M92IvKEiGwDvsKEqRmmqq09OV9VC4CiyBlbMHGgN4nIUyIy0Er2DXBMRDYDS4DxqnqsgtdTLvn717DCoXQJacexjHwGdrObe15lzx644AJ44gmYN8/ss7s3XsPT9s5q4FPgAeC7yjg+UNUvMUHcXfc97rKumNA6HoXXqSy/bHifLD8/0vL7EBHkoF9726LKa3z+Odx+u3HgN2eOsV228SoViT6Z41VJqollfywnwB9W7GjJlZ0b20YY3uLtt2HkSEhIgNmzoa09aac6KFWhReQmVf3Q2hxS2iiwqr7nDcG8Qk4aPznTODuwKT/l+DPQHgzzHlddBQ8/DP/85xnj5L4mUFYNPRwoUuhRpaRRTIzoWkHyb4vZERjIublnERsRRO82tlVVlfLtt8bz5vvvQ5Mm8OKLvpbojKNUhVbVK1zWa72ZJMCaXV8A8MsfnRmc0NR2y1tVFBaamvipp4yZ45EjtpWUj/B0lNvtlE8RWVG14niXVUc3EKKQlhlvf3uuKg4dgiuugCefNKFnVq60ldmHeDoo1qGU/bXHeDg/m9WFabTQaMJi6tG1ue1k7rRRNSFnEhPNINidd9qfpHxMedEni1wMBbqsFxGP+aZcKzi6ezG/BThoeTSefu1j7amep4PTaRaHA/7zH2Mh1bWrr6Wyofwaen8p64oJKVtrAtet3mn6z3vSe3B/qygfS1OLOXbMfFtu39641e3Rw9cS2bhQXuSMx8D0lVX1i+oRyTusP76NEKdyOKc1PeJtha4UK1bAkCGm33z11b6WxsYNZX2H7qOqP1mb6SLi1oWEqv7gFcmqmO25x2haGEhEbD1iwm3n+RVCFf79bxg/Hlq0MNErzj3X11LZuKGsGvptTgyGfVBKGgXiqlQiL6CFhWwnn5Y5MZzdyrZ7rjBJSfCPf5ha+Z13oEEDX0tkUwplfYfu4LLeorR0tYHDhxJJ9fejIKchPVvZD6PH7NljAsK1agWrV5tvzPZgYo2mUsaoInIBUKCqy6tYHq+wY5/pOaRnx9PTBzV0fn4++/btIyenFk2HT083IWiOHIGwMON9c+tWX0t1xhEcHEzz5s0JCPDMxNdTn2JLgcdU9UcReRiYABSIyBRVfb7S0lYT2w9vACDHkUCzyOr3W7Vv3z4iIiKIj68FnlEKC03NnJlpauZWrcDDh8mmalFVjh07xr59+2jVqpVH51TE0X5RbfwXjJug84B7KyqkL9ievpuGBU6aN/aKA5RyycnJITo6uuYrc3a28cKZnGzcAp11lq3MPkREiI6OrlDLztMmtx/gFJHWgENVN1kF1orvPztyk2mU6+DseN/FEa7xygzGeV9hoYm/XK+er6WxoeLPjacK/TMwBWiKcXSApdxV7lGkqskvzOc38umR24AOTezA4KdQWGia1/XqmdHrevVsv9i1GE+b3MOBHIzjviesfR2B/3hBpipl96FECkTQ3Eac3eTMrXXCw90EEcjJMQNdO3YYv19QpjIfOHCAwYMHl3o8JSWF//3vfx6nL8nw4cNp1aoVCQkJdOvWjUWLFnl8bnUwdepU3nuvhlsLq2q1LMCVmBfCTmBiGekGY75v9ygvz3PPPVfL48df3tTOMzrrkH8+qLn5heWm9wabN2/2SUEbQQMAACAASURBVLmuhIWFnbzj2DHVtWtV161TTUmpkjJ2796tnTp1qvT5d9xxh86ePVtVVRcvXqxt27atErny8/OrJB9f4e75AdaoG53wdJTbgQlTcxvGQf5+TKzoyaqaX9a51vnlhsKx0kVg/Jat9EQuT8jNTQUgNDyWQIfvA589uWATmw+kVWmeHZvW44lrO3mWWBX27oXDh9mTmsqIp5/myLFjxMbG8s477xAXF8euXbu45ZZbKCwsZMCAAbz88stkZGSQlJTENddcw8aNG9m0aRN33nkneXl5OJ1O5s6dy2OPPcauXbtISEjg8ssv57777itOX1hYyIQJE/jmm28QEUaNGsWYMWNKFbNXr17s33/CfGDt2rWMGzeOjIwMYmJimDFjBk2aNGH16tXcddddhIWF0bdvX7766is2btzIjBkz+OKLL8jJySEzM5PFixfz4osv8sknn5Cbm8v111/Pk08+SWZmJkOGDGHfvn0UFhby2GOPMXToUCZOnMj8+fNxOBz079+fl156iUmTJhEeHs7DDz9MYmIio0ePJisrizZt2jB9+nQaNGhAv379OO+881iyZAkpKSm8/fbbXHBB9bkT8PQJfx64GngQE9bmQWAAMNnD84tD4ahqHlAUCqck/wRewDTvq4S8/CwAmjaoFeN33kfENKsbN+b+V17h9uHD2bBhA7fccgsPPPAAAGPHjmXs2LGsXr2apk3d241PnTqVsWPHkpiYyJo1a2jevDmTJ0+mTZs2JCYm8mIJbyXTpk1j9+7drFu3rri8svj666+57rrrAPMdf8yYMcyZM4e1a9cyYsQIHnnkEQDuvPNOpk6dyvLly/Ev0V1Yvnw57777LosXL2bhwoXs2LGDVatWkZiYyNq1a/nhhx/4+uuvadq0KevXr2fjxo1ceeWVJCcn8+mnn7Jp0yY2bNjAo48+eop8t99+O88//zwbNmygS5cuPPnkk8XHCgoKWLVqFVOmTDlpf7XgrtouuWDC2MSU2BcL7Pfw/MHAWy7btwGvlUjTHZhrrS+llCY3cDewBlgTFxdXbnPlo4UTtPOMzjr1808q0MipWmpMkzstzWw4naqqGh0drXl5eaqqmpeXp9HR0aqqGhUVVdxMTU1NLW6uuzapP/jgA+3YsaNOnjxZt2/ffsrxkts33HCDLly4sEwZ77jjDo2Pj9dWrVppWFiY/vrrr6qq+uuvv2pERIR269ZNu3Xrpp07d9bLL79cjx8/rq7PwPr164vLe+edd3T48OHFx/76179qy5Yti/No06aNvvXWW7pt2zaNj4/Xv/3tb/rDDz+oqmmid+3aVUeMGKFz587V3NxcVVV94okn9MUXX9SUlBRt0aJFcd47d+7U7t27q6rqRRddpMuWLVNV1YMHD2qbNm3KvGZPqEiT29Ma2h9wltjnxH2IG3eUGQpHRPyAV4C/lpeRVjAUztEM07xtGePVYJY1F6fTNLGdTjh40Owr5VNIRT6R3HzzzcyfP5+QkBCuuOIKFi9eXGZ6VfUo/xdffJGdO3fy9NNPc8cddxSf26lTJxITE0lMTOTXX39l4cKFRS/4UgkLOxHrW1X5+9//XpzHzp07ueuuu2jXrh1r166lS5cu/P3vf+epp57C4XCwatUqbrzxRubNm8eVV7oNs1YqQUHG+Mff35+CgoIKnXu6eKrQc4D5InKpiJwlIpdhPl/N9fD88kLhRACdgaUikgScb5V32sa22XnZADSIOCVMVt0nNxe2bTPmjiLQps1Jh3v37s1HH30EwAcffEDfvn0BOP/885k71/y1RcdL8ttvv9G6dWseeOABBg4cyIYNG4iIiCA9Pd1t+v79+zN16tTiBzw5OblUsf38/Bg7dixOp5NvvvmG9u3bc+TIEZYvN3Ob8vPz2bRpEw0aNCAiIoIVK1aUKSvAFVdcwfTp08nIyABg//79HD58mAMHDhAaGsqtt97Kww8/zC+//EJGRgapqalcddVVTJkyhcTExJPyql+/Pg0aNODHH38E4P333+eiiy4qtezqxNPv0OMxn6veBppglPEjwNMOQnEoHMyA2jDg5qKDqpoKFLvgtKaaPqyqazzMv1RyC3MBCAs5wxQ6N9fM+nI6oXVrsrKzaR53wjBu3LhxvPrqq4wYMYIXX3yxeFAMYMqUKdx6663861//4uqrr6Z+/VPdNX388cfMnDmTgIAAGjduzOOPP05UVBR9+vShc+fODBgwgPvuu684/ciRI9m+fTtdu3YlICCAUaNGcf/995cqvojw6KOP8sILL3DFFVcwZ84cHnjgAVJTUykoKODBBx+kU6dOvP3224waNYqwsDD69evnVlYwL5QtW7bQq1cvwHzGmzlzJjt37mT8+PH4+fkREBDA66+/Tnp6OoMGDSInJwdV5ZVXXjklv3fffbd4UKx169bF987nuGuHe2MBrgK2A7uAR6x9TwED3aRdShV9tnrivRu184zOuvtgcrlpvYVP+tBOp+revarZ2RU+NTMzU51WP/vDDz/UgQMHVrV0VUZ6enrx+nPPPacPPPCAD6XxDlX22UpEzsLUyp2BX4ARqvp7JV8cZYbCKbG/X2XKcEeeM5dApxJyJjh7z8uD3383TgiCgqB580pls3btWu6//35UlcjISKZPL+lOrubwxRdf8Nxzz1FQUEDLli2ZMWOGr0XyKeU1uV/DNJFfwjSRpwA3eFuoqiTPmU+gKkF1PeRNWhr89ptpYmdnG4WuJBdccAHr16+vQuG8x9ChQxk6dKivxagxlKfQ5wItVDVbRJYAtc4gNl/zCVTqbgwrVRNE/cABE3KmTRsIqX4TUZuaQXkKHaiq2QCqmi4ite5JyXfmEwAE1YBZYl7h0CGjzFFRxruIbVhxRlOeQgeJiGs/N6TENqr6VNWLVXXkUUCA1hLzxYrgdBovIrGxxmY5Ksp2D2RTrkJ/Arh6BZhTYrvsL/s1gAItIEDr0IOuamrlY8egQwdTI0fbjg9tDGW2Q1X1tnKW26tL0MqSjxOH1pHmdkEB7NwJ+/ZVOESriHDbbbe5ZFVAbGws11xzTbnnFpleJiUlMWvWrOL9a9asKZ7/PWPGjDK/K1c1SUlJhISEkJCQQMeOHRk9ejROZ8nJjN6npMmor6kjT3rp5FOIw+MZqjWYzEzYvNmMZrdoAa1bV6i/HBYWxsaNG8nONjPnvv32W5o1a1YhEUoqdI8ePXj11VcrlEdlcTeFssgQZMOGDWzevJl58+Z5lJeqVpnyl6XQhYWFVVJGRaiU18/aRAFOgmtSDf3VRDj4awVPUsjKMs3tkBDYUEKRG3eBAeUbvg0YMIAvvviCwYMH8+GHH3LTTTcVT190NQ0E6Ny5M59//jnx8fHF50+cOJEtW7aQkJDAHXfcQffu3XnppZf4/PPPTypnwYIFPP300+Tl5REdHc0HH3xAbGws7du35+effyY2Nhan00m7du1YsWIFqsro0aP5/XczxWHKlCn06dOHSZMmceDAAZKSkoiJiTnpZeKKw+Ggd+/e7Ny5E8CtmWRSUhIDBgzg4osvZvny5cybN48tW7bwj3/8g8LCQmJiYli0aBGZmZmMGTOGX3/9lYKCAiZNmsSgQYOYMWMGn376Kbm5uezevZubb76ZJ554gokTJ55kMnr11Vfz5JNP0qRJExITE9m8eTMvv/xy8bf8kSNH8uCDDxbL07dvX37++WeaNWvGZ599RshpfqGoQU+6d8hHcVBLR35VzYIYRQ4NA7/KX8uwYcP46KOPyMnJYcOGDZx33nkVOn/y5MlccMEFJCYm8tBDD5Warm/fvqxYsYJ169YxbNgwXnjhBfz8/Lj11lv54AMTs+G7776jW7duxMTEMHbsWB566CFWr17N3LlzGTlyZHFea9eu5bPPPitVmQGysrJYtGgRXbp0KdVMEmDbtm3cfvvtrFu3jtDQUEaNGsXcuXNZv349s2fPBuCZZ57hkksuYfXq1SxZsoTx48eTmZkJwKpVq/jggw9ITExk9uzZrFmzxq3J6KpVq3jmmWfYvHkza9eu5Z133mHlypWsWLGCN998k3Xr1gGwY8cO7rvvPjZt2kRkZGTx/PnToc7X0PnirFkK7UFNCpgaedcuCA09xaiisnTt2pWkpCQ+/PBDrrrqqirJ0x379u1j6NCh/PHHH+Tl5RW7oB0xYgSDBg3iwQcfZPr06dx5552AUe7Nm0/4ukhLSys28hg4cGCptVZRzSgiDBo0iAEDBvDwww+zcOFCunfvDkBGRgY7duwgLi6Oli1bcv755wOwYsUKLrzwwmLZoqKMvfzChQuZP38+L730EmA8tha1HC6//HKirQHIG264gWXLlhXbbLvSs2fP4nyXLVvG9ddfX2z5dcMNN/Djjz8ycODAYndLAOeeey5JSUkVvtcl8VihReRijFFFI1W9TkTOASJU9fvTlsKL5Ak4atN7SxWOHjVTOB0OaFi1Zp8DBw7k4YcfZunSpRw7dsLHo8PhOKlfeTpBAcaMGcO4ceMYOHAgS5cuZdKkSQC0aNGCRo0asXjxYlauXFlcWzudTpYvX+5WcV1NIEtSVDO6opaZ5F/+8peT9iclJZ1iTunuU6aqMnfuXNq3b3/S/pUrV56SvrRPoSXLKY0gl9l8/v7+xeMbp4NHTW4RuRczp3svcLG1Ow945rQl8DJ5orVHoQsLYfdu4+g+IsKEnomoWk+lI0aM4PHHH6dLly4n7Y+Pj+eXX34B4JdffmH37t2nnFuWeaQrqampxQNu77777knHRo4cya233sqQIUOKPYz079+f1157rThNSSWtCKWZSZakV69efP/998XXWWTOecUVV/Cf//ynWBGLmsdgBhKTk5PJzs5m3rx59OnTp9x7cuGFFzJv3jyysrLIzMzk008/9apLIk/70H8FLlPVpznh6GALcLZXpKpC8kVwSC1xFl9YaELQNG3qNSf3zZs3Z+zYsafsv/HGG0lOTiYhIYHXX3+ddu3anZKma9euOBwOunXr5taksIhJkybx5z//mQsuuICYmJiTjg0cOJCMjIzi5jbAq6++ypo1a+jatSsdO3Zk6tSplb6+/v37c/PNN9OrVy+6dOnC4MGD3SpcbGws06ZN44YbbqBbt27F88Efe+wx8vPz6dq1K507d+axxx4rPqdv377cdtttJCQkcOONN9KjRw+io6OLTUbHjx9/SjnnnHMOw4cPp2fPnpx33nmMHDmyuDvgFdyZYJVcgMOAv7WebP0GAX94cr63Fk/MJ7u900kfeuPKctN5k3LNJ1NSit0CaUGB9wXyIatXr9a+ffv6WowK88477+h9993nk7K94YJoGfBwiX33ATW6/1xQmE9hTa6hnU4TqnXHDjPzC+r0XOzJkydz44038txzz/lalDqLp53LMcDnIjIKiBCRTZg+tPeGSquAvDzTjwrwC/SxJG7IyTGj2NnZ0KTJGTF9c+LEiUycONHXYlSK4cOHM3z4cF+LUS4eKbSq7heRc4FemADve4Hlqlr9U2EqQJFP7gC/ytsGe4WUFGO77Odn+sqluM2xsakoHk8sUVWnqv6kqh+q6rKKKrOIXCki20Rkp4ic8poWkXEisllENojIIhFpWZH83ZGZbQZDAvxrmLcSh8N8Xz77bFuZbaoUTyNn7KYUyypVbe3B+Z5EzliH8SOWJSL3YBzun5YrivTsFAACHTVAoXNzITXVfFcOD4f27W1zR5sqx9M+9MgS200w/eoPPTy/OHIGgIgURc4oVmhVXeKSfgVwq4d5l0pGtvHJHejwsV+GrCxjWAEmwmNAgK3MNl7Boya3qi4qscwErgPu8LCcZph+dxH7rH2lcRfwlbsDInK3iKwRkTVHjhwps9CsbDMo5jOFzs+Hv/0NjhwxPr46dvRZAHV30Sc9iaZYllnks88+W+p5GRkZ3HPPPbRp04bu3btz7rnn8uabbwInmz5269aN3r17s23bNgCWLl2KiPD2228X57Vu3TpEpHg6pk3pnI5xRjZQbnPboszIGSclFLkV6AG86O64ViByRnauUeiggFAPxaxCVOHqq+HFF00Tu0OH03Lc5w1Gjx7N7bdX3qS9LIUeOXIkDRo0YMeOHaxbt46vv/76JOf6RdM2169fzx133HFSXl26dOHjjz8u3v7oo4/o1q1bpeU8k/C0D13S3W4oJnjdQg/LKS9yRlE5lwGPABepaq6HeZdKVp6xkgkOKH0+sNcQgdtugzvvNJ+k/My78/lVz7M1uWp9LXaI6sCEnhMqfJ6ryWRpURzBxHm+8sor2bVrF9dffz0vvPACEydOJDs7m4SEBDp16lQ8LxuM0cSqVauYNWsWftZ1x8bGMmGCexnT0tJo0KBB8XZcXBxpaWkcOnSIhg0b8vXXX3vVmKQu4Wkf+qwS25mYQa4ZHp5fZuQMABHpDrwBXKmqp06+rQS51nfooMBqUujCQnjySWjbFm6/3Sg0mAgWNZw777yTadOm0bt371O+FScmJrJu3TqCgoJo3749Y8aMYfLkybz22mtu511v2rSJbt26FSuzO4ospdLT08nKymLlypMjCA8ePJjZs2fTvXt3zjnnnJMMGWxKp1yFtkaovwU+UdVKmeCoaoGI3A98gwl8N11VN4nIU5gpbPMxTexwYLZlxfK7qg6sTHlF5FhxrYKDTu0/VjkHD8LNN8OSJXDffUah3VCZmtTbpKSkkJ6eTu/evQETiM7VacGll15aHGKmY8eO7NmzhxYtWrjNyx3PPPMMs2fPLo4lBSdbSn388cfcfffdfP3118XnDBkyhKFDh7J161Zuuukmfv7559O+zjOBcvvQ1vfm/1RWmV3y+VJV26lqG1V9xtr3uKXMqOplqtpIVROs5bSUGSCnwCh0WFDVWiydwtKl0L07rFgB77wDLpZDtQEtJ4pjSTO/8iIqduzYkfXr1xebYz7yyCMkJiaSluY+0P3AgQOLnRAU0bhxYwICAvj222+59NJLPbkMGzwfFPtCRGpdJybXUujQEC9O3ti+HS67zEwQWbUKasH0wJJUJIqjKwEBAeTn55+yv23btvTo0YNHH3202K9WUeA3dyxbtow2bpw4PPXUUzz//POnBHK3KR1P+9B+wP+JyDLM56fif0ZVR3hDsKogr6iGDq5X9ZkXFJgZX+3awYwZMGhQldsuVyVZWVk0d4l1NW7cuJOOexrF0ZW7776brl27cs4555w0KAbw1ltvMX78eNq2bUtUVBQhISE8//zzxceL+tCqSmBgIG+99dYp+Rd1AWw8R8prbgGIyD9LO6aqj5V2zNv06NFD16wpPeLsP2fewieFG/ii/xfENYkrNV2F+fln00eeNQt69iw3+ZYtWzj77JptOp6RkVH8rXry5Mn88ccf/Pvf//axVDbg/vkRkbWqekr89PKiT95kzd32mdKeDvlFsaHDqqjJrQqvvAITJkBcHATWQCuuSmJHcawblNfkfgPPp3fWOPKcefiLEh5SBZ+tUlJM//izz+D6683gVx0yrLCjONYNyhsUq9UTjguceQSqEuiogkGVN9+EL74wNfTcuXVKmW3qDuXV0P6Wt89SFVtVF1etSFVHUSjZSgeqUzWRHZs1g3Hj4IoroGvXqhXSxqYKKTf6JMbbZ2kaoXg+n7vaKVLoSpGeDqNGmW/MGzdCTIytzDY1nvIUOtMTe+eaiok8WYkTN2yAP//ZBIZ75hkTqtXGphZQp0Ph5FNIQEXiWqnC9Olw3nmmhl68GCZOLDasqO0cPHiQYcOG0aZNGzp27MhVV13FtGnTPIpAaVM7qNODYvkU4qhobOjPPoM+fWDdOrjoIu8I5gNUleuvv55+/fqxa9cuNm/ezLPPPsuhQ4d8LZpNFVJmk1tVa+7UJw8ooJAAT95JW7eaeMvx8fDBByYwnDenG/brd+q+IUPg3nuNdxN3poLDh5vl6FEYPPjkY0uXllvkkiVLCAgIYPTo0cX7EhISSElJYdGiRQwePJiNGzdy7rnnMnPmTESEp556igULFpCdnU3v3r154403EBH69evHeeedx5IlS0hJSeHtt9/mggsuoLCwkAkTJvDNN98gIowaNYoxY8awdu1axo0bR0ZGBjExMcyYMYMmTZpU5I7ZeEjdaEuWgkfB3mfNgh49jIUUGGcEdXDucJGyumPdunVMmTKFzZs389tvv/HTTz8BcP/997N69eriuNKuFlgFBQWsWrWKKVOm8OSTTwIwbdo0du/ezbp169iwYQO33HIL+fn5jBkzhjlz5rB27VpGjBjBI4884v0LPkOpJUGfKkeBKKFainLm5MCDD8Ibb0DfvjBtWvUJVlaNGhpa9vGYGI9q5IrQs2fP4nneCQkJJCUl0bdvX5YsWcILL7xAVlYWycnJdOrUiWuvvRYwURTh5KiJ3333HaNHj8bhMI9VVFQUGzduZOPGjVx++eWACYJu187eo04rdD6lhJLdtw+uvRYSE800zqefNoYWdZhOnToxZ84ct8fcmUfm5ORw7733smbNGlq0aMGkSZNOikhZdI6rOaW6ieioqnTq1Inly5dX9SXZuKFuN7kFAsSNokZGQlgYLFgAkyfXeWUGuOSSS8jNzS121AewevVqvv/efTSjIuWNiYkhIyOj1JeBK/3792fq1KnFCp6cnEz79u05cuRIsULn5+ezadOm070cm1Ko4wqtBBQ1QvLy4LnnIDPT9JN//BHOoM81IsKnn37Kt99+S5s2bejUqROTJk2iadOmbtNHRkYyatQounTpwnXXXcef/vSncssYOXIkcXFxdO3alW7dujFr1iwCAwOZM2cOEyZMoFu3biQkJNjeR7yJuwh23liAK4FtwE5gopvjQcDH1vGVQHx5eZYXfbLX2x31gWmXqCYlqfbsqQqqH3xQ5jneoNzokzY2ZeCN6JOnhUvkjAFAR+AmEelYItldwHFVbQu8AjzPaZInQpcNKcY90NatMGeO8ftlY1NHqa4md3HkDFXNA4oiZ7gyCHjXWp8DXCqVtqoAZ2Ehg5YmM/K1X6BlS1i7Fm68sbLZ2djUCqpLoT2JnFGcRlULgFTglBirnkbOyMrJYGWncH6+vK3xMNK27elew2mhHniGsbEpSUWfm+pSaE8iZ3gUXUM9jJwRHlafz/+2nfO/2mpmfvmQ4OBgjh07Ziu1TYVQVY4dO0ZwsOfBFqvre40nkTOK0uwTEQdQH0jmNPGrAbO+mjdvzr59+ygvFpeNTUmCg4NPcu5YHtWl0OVGzgDmY4LfLQcGA4u1jlRpAQEBtGrVytdi2JwBVItCq2eRM94G3heRnZiaeVh1yGZjU5eotilSqvol8GWJfY+7rOcAf64ueWxs6iJ1eqaYjc2ZhkeO9msqInIE2FNOshjgaDWIUx41RQ6wZXFHTZEDPJOlpaqe8pmnViu0J4jIGnUTYeBMlQNsWWqyHHB6sthNbhubOoSt0DY2dYgzQaGr0RVJmdQUOcCWxR01RQ44DVnqfB/axuZM4kyooW1szhhshbaxqUPUCYUWkStFZJuI7BSRiW6OB4nIx9bxlSIS70NZxonIZhHZICKLRKSlr2RxSTdYRFREvPLZxhM5RGSIdV82icgsb8jhiSwiEiciS0RknfUfuXGSXiVyTBeRwyKysZTjIiKvWnJuEJFzPMrYnRuT2rRg5obvwgTNCwTWAx1LpLkXmGqtDwM+9qEsFwOh1vo9vpTFShcB/ACsAHr46J6cBawDGljbDX34/0wD7rHWOwJJXpLlQuAcYGMpx68CvsKYFZ8PrPQk37pQQ1e7N5TTkUVVl6hqlrW5AmNK6g08uS8A/wReAHLcHKsuOUYB/1XV4wCqetiHsihQz1qvz6lmvlWCqv5A2ebBg4D31LACiBSRch2a1wWFrjJvKNUkiyt3Yd7C3qBcWUSkO9BCVT/He3hyT9oB7UTkJxFZISJX+lCWScCtIrIPY0w0xkuylEdFnyWgbjjarzJvKNUki0kocivQA/BWRLwyZRERP4wzxuFeKt8jOSwcmGZ3P0yL5UcR6ayqKT6Q5SZghqr+S0R6YUx6O6uqs4plKY9KPbN1oYauiDcUqtIbSiVlQUQuAx4BBqpqrhfk8ESWCKAzsFREkjD9tPleGBjz9P/5TFXzVXU3xt3zWVUsh6ey3AV8AqCqy4FgjLFEdePRs3QK3ujwV+eCebv/BrTixEBHpxJp7uPkQbFPfChLd8zAzFm+vi8l0i/FO4NintyTK4F3rfUYTFMz2keyfAUMt9bPtpRIvPQfxVP6oNjVnDwotsqjPL35UFXXghkR3G4pyiPWvqcwNSCYt+xsjBP/VUBrH8ryHXAISLSW+b6SpURaryi0h/dEgJeBzcCvwDAf/j8dgZ8sZU8E+ntJjg+BP4B8TG18FzAaGO1yT/5ryfmrp/+NPfXTxqYOURf60DY2Nha2QtvY1CFshbaxqUPYCm1jU4ewFdrGpg5hK3QNQ0RmisgkX8tRHpbF0gVlHF8oIrdUp0w2tkJ7DRFJEpFsEclwWZr6SJaZIpJnyZBsKVu708lTVdur6o9W/k+LyIwSx/ur6genU0ZJRMRhmXlmWteyT0RetKaxenL+ZdasuDqLrdDe5VpVDXdZvGK54yHPqmo4ZjphMjDdh7KcLp2sa7kEuA0TE80GW6GrHRHxE5E5InJQRFJEZKmInF1K2oYi8qWVLllEfnA51lxEPhWRIyKyW0Tu86R8Vc3EzFLqbOUTbBnS/yEi+0XkZREJ9KD8fSLST0SuAf4G3GLVmmut48tEZLiIhIhImoh0cDm3sdV6iba2B4rIequcZSLS2cNr2Q78DCS45D1SRLaISLqI7BKRkdb++sACIM6lxdTQ+j/+YaU9KiIfiUgDT8qvidgK7Rs+xxgfNAY2Au+Xkm48Zu5xrJX2MQAR8bfyWI0xqbscGC8il5ZXsIhEYCJ/rrN2PY6x+uqKmWfeB/h7WeW7osb08gXgA6sVcm6J49nAtVMO3AAAA0xJREFUPIwVUxFDgUWqekxE/gS8CYzEmLROBz4reqmUcy1nW/LudNl9CDMPuh7Gzvo/ItJVVVOBa4HfXVpMh4FxVvoLMQYQmcCr5ZVdY/HWnNkzfQGSgAwgxVrmlZIuBmMWF2ZtzwQmWevPAv8HtClxTh/gtxL7HgPeLKWMmRgHBimY+cPzgFbWsT24zFfGPNw7yyrfOrYP6GetP40xOXQ9vowTRg5XAttdjq0EbrbW3wSeKHHuLqCPmzId1r1KwyieWtcWWMb/8Dlwn7V+GSU8kAA7gItctlsAuYCfr5+hyix2De1drlPVSGu5DkztKiIviMhvIpLGidrFnYneZIzCLbKahOOt/S0xTceUogXT7G1chiyTLTmaqOp1aswUAZpwcnywPZwwpC+t/IryHcbjxrki0gboBHzmci0TSlxLE8o25u+KMf+8GegFhBYdEJFrxPiNS7by6k/Z5o9xwAKXsn/FvCgaVupKfYyt0NXP7RiLn0swdtltrf2nGLSrapqqPqSq8cB1mAf/Iox54Q6Xl0Wkqkao6rWVkOcPjFIVEQfsL6f8U0QtqwA1XmJmY5rdN2NsnzOtw3uBJ0tcS6iqflJOnk5V/RBYAzwKICIhGBdTzwGNVDUSWMiJe+tOzn3A5SXKD1bVg2WVX1OxFbr6icA06Y5hapZnSksoIteKSBsREYzbpEJrWQ7kichfrUEtfxHpIiLnlpZXGXwIPC4iMSISi2m6zyyn/JIcAuKtdKUxC9N3vtlaL2IacJ+I/EkM4Va5YR7K/xww2pI9CGPnfAQotAbsXMcVDgEx1jhCEVOBZ0UkzrrmhiIy0MOyaxy2Qlc/72CM5g8AmzCjtKXRHliM6Yv/BPxbVZdZNd5VGKd3SZjQo29wwrldRXgSY/v7K7AB0799rqzy3eTxMUaRkkX+v507NkEgCKIw/NuEndiJoU0I9nJViKlYgKCRgQVcBwamwhjMXShcKI//C4+F2eTt3t4xu7r/qHMFPvQHtsv8sKpu9O2nA/Cie5V3SydfVQ96gTtUX1m0B070r7ktfYaexz6BIzBOr9hrug/7TB8r3tM8N0vr/xv7oaUg7tBSEAMtBTHQUhADLQUx0FIQAy0FMdBSEAMtBfkCphO6h1uocgMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 252x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(3.5, 3))\n",
    "for idx, label in enumerate(['Logistic Regression', 'Multilayer Perceptron', 'Light GBM']):\n",
    "    fpr = mdl_res[idx]['mean_fpr_minute']\n",
    "    tpr = mdl_res[idx]['mean_tpr_minute']\n",
    "#     plt.plot(fpr, tpr, label=f'{label} (auc={metrics.auc(fpr, tpr):.3f})')\n",
    "    plt.plot(fpr, tpr, label=label)\n",
    "\n",
    "plt.plot([0, 1], [0, 1], 'r--', label='Chance')\n",
    "plt.xlabel('False Positive Rate', size=12)\n",
    "plt.ylabel('True Positive Rate', size=12)\n",
    "plt.title('ROC for Minute-wise Prediction', size=12)\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.savefig('../archive/Model_Minute_ROC.png', dpi=600)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-07T03:48:50.028274Z",
     "start_time": "2020-02-07T03:48:49.708103Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPQAAADQCAYAAAAwEkzeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO2deXxU1dnHv0/2kIQkkLDvCLKpoChQF7SiBVS01qqgvuKG+qpttVq1raitu9Uu6utStVituFBURBTcEFFUQEFZZJUlLAmQQPZlMs/7x7mBMSSZSzJ7zvfzmU/u3Llzzy/J/Oace+55nkdUFYvFEhvEhVuAxWIJHNbQFksMYQ1tscQQ1tAWSwxhDW2xxBAJ4RbQEnJycrRXr17hlmGxhJylS5fuVtXc+vuj2tC9evViyZIl4ZZhsYQcEdnc0H475LZYYghraIslhgiJoUXkeREpEJEVjbwuIvIPEVkvIt+KyNGh0GWxxBqh6qGnAWObeH0c0M95TAGeDIEmiyXmCImhVXUBUNjEIWcD/1bDF0CWiHQOhTaLJZKoqfVSVFbd7PdHyix3V2Crz/M8Z9+O8MixWAKL16vsKq1i294KduytpLCsin0VNRSW1VBUXk1ReTVxa9dxxpxpzL/pzzx26chmtRMphpYG9jUYBiYiUzDDcnr06BFMTRbLj/DUetlXYQy4r6KGimovlTW1VHpqKa30UFLpYU9ZNUVl1RRX1lBcWWNMW1rN7tJqqmu9B50zLSme7LQkRm/9lt//ayokJtI119NsjZFi6Dygu8/zbsD2hg5U1WeAZwCGDx9uYz8tzcJT66Wsqna/6YrKqyksq6asqpbyao/zvIbdpVXs2FdBXlEFe8tr/J43MV7IbpNE29REMlMTyU1P5vCObcnNSKZrVgpdslLpnJlKTkYSmamJJCfEw1dfwfG3wOGHw+zZjGzBYqlIMfQs4HoReQUYAexTVTvctrimylNLQXEVhWXVlFR6KK6sYW+56SV3l5ihbn5xJbtLTQ9aUtV0LxgfJ2SlJpKTnkynzBSO6pZFbkYyWamJZKcZM7ZJSiA1MZ6khDjSUxJom5JAenICIg0NOJtg+HC45x649lpo27YFf4UQGVpEpgMnAzkikgfcCSQCqOpTwBxgPLAeKAcuC4UuS+RSU+ulqLya8qpaSqs87C6tYue+SrYUlrO7tMoMb0ur2V1mTNxU75mcEEe37FQ6ZaYwrEcW2W2SyGqTSEZKIhnJCbRNTSS7TSLt05NIT04kNSmejOQE4uIO0ZiHQnEx/OpXcPfd0LMn3HprQE4bEkOr6kQ/rytwXSi0WCKH0ioPa3aWsGl3GZv3lLHB+bmtqIKiRgyaECe0T08iIyWRdmlJDOiUQfu0ZHLSk+mclUL7tCTSkxPIbJNI25REpyeNP/ReM5hs2gRnnQWrV8O4ccbQASJShtyWGKXKU8vqHSVs3FXKpj3lbC00j+17K9i+r3L/cXEC3bLb0CsnjSO7ZdExI4V2aYmkpySQmphAbkYSHTJS6JyZQkJ8FC9wXLQIzjkHqqrg3XfhtNMCenpraEvA2Ftezdr8UjbuKuX7nSV8s3Uvq7cX75/dFYEumal0y05lZJ/29MpJY2DntvTNTaNbdhuSEqLYqG74+GPTI3frBp98AgMGBLwJa2jLIVNa5WFrYTlr80v4Zste1uaXsL6glIKSqv3HtEmK54iumVx2Qi+GdsuiX8cMurdLNbO6rZXhw2HyZDMBlpMTlCasoS2NUlrlYdX2YlZt38e6glI27ipjw64fGzc1MZ7+nTI4oV8Oh3fMoH+nDA7LTadrVmpwJ5WihYoKuPdeuP12yMiAp54KanPW0Jb9lFd7WLm9mPdX5fPR9wVs2FVKXZbnzNRE+uSmcWK/XPrkptGzfRv65KTTv2N6dF/TBpMdO8z18uLFpnc+55ygN2kN3UopKKlk1fZilm3dy4ptxawrKGFLYTmqZnHEyD7tOevILhzRrS2Du2TSISM5smaKI51ly8xMdmEhzJwZEjODNXSrwetV1haU8Pby7bzx9bb9M8wi0CcnjcFd2nLusG4M7JzBiD7tyUxNDLPiKOb99+HnP4esLFi4EIYNC1nT1tAxSq1X+W7bPj5bv5vFmwr5enMRxZUe4gRG98/lihP7MKhzW47olkl6sv0YBJS+feGkk+DZZ6FLl5A2bf+TMcT2vRXMW7mTzzbsYfGmwv2rp/p3TOeMIztzTM92nHBYDp0yU8KsNAaproYXXoArr4Q+fWDOnLDIsIaOctYXlDJr2TY+/L6AlduLAejRrg2nDezICf1yOLFfLu3SksKsMsbZswd+8Qtzb7lvX/jpT8MmxRo6CskvruSdb3fwznc7WLq5iDiB4T3bcevYAZw+uCN9c9PDLbH18P33ZvJryxZ48cWwmhmsoaOGiupa5q3ayYyleXy2fjdehQGdMrht3ADOPborHTLsMDrkfPSR6ZkTE8328ceHW5E1dKRTWFbNs59u5MVFmymp8tA1K5XrTzmMs4d1tT1xuBGB3r3NbakIKfhgDR2hrC8o4YXPNzNjaR6VnlrGH9GZi0f0ZETvdnYFVjiprTVrsseMgVNOgSVLIC5yFtYcsqFFpIOqFgRDTGvHU+tl1vLt/HvRZpZt3UtSfBxnD+3ClJP60K9jRrjlWYqLYdIkeOcds3DkqKMiyszg0tAikgk8BpwP1AJpInIWMFxV7wyivlZBZU0tM5bm8fSCDWwtrKBfh3T+MH4g5wzrSm5GcrjlWeDHMcxPPWXMHIG47aGfBMowebO/dfZ9CTyCyT5iaQbf7yzm1cVbmbVsO3vKqhnaPYs7zxzMqQM72GWWkURdDHN1Nbz3nhluRyhuDT0G6Kaq1SKiAKpaICIdgyctdtm8p4xH31/LW8u2kxQfx6kDO3DJyJ6M6tveGjkSWb7c5PqaPdsk8otg3Bq6GGgH7KzbISLdgfxgiIpVtu2t4NF5a3lz2TYS44VrT+7LlBP7kG0XfkQeXi+sWQMDB8I118All0BaWrhV+cWtoZ8HXheR3wNxInIscD/wdNCUxRBVnlqemr+RJz9ZjypM/kkvrh7dx947jlTKy00igjlzYOVKk/MrCswM7g19P1ANPAekAC9jzPzXIOmKGT7fsJs73lzBhl1lnHFEZ24fP4Bu2W3CLcvSGDt2wNlnm9tRDz0EUVbMwa2h26vqX4C/+O4UkRxgt5sTiMhY4O9APPCsqj5Q7/UewAtAlnPMbaoanhXuAWDl9n386e1VfPlDIV2zUpl22bGcfHiHcMuyNEVdDHNREbz5JkyYEG5Fh4xbQ28EGsoAvhZzbd0kIhIPPAGchqmSsVhEZqnqKp/D/gi8pqpPisggTK7uXi71RRSzlm/ndzOWk5GSyNQzBzHxuB6kJrXiXFrRwrRp5ufChTB0aFilNBe3d8UPmnoVkXTg4GI9DXMcsF5VN6pqNfAKpuKkL8qBL41MGimFE+k8s2ADv5r+DUd2zeLdX5/I5Sf0tmaOZFRh1y6z/fDDZqgdpWYGPz20iPyAMVqqiGys93IO8F+X7TRUXXJEvWPuAuaJyA1AGuZWWUOaIrJYnaryxMfr+cu8tZxxZGf+ev7Q2E9LG+1UV5vyMx99BF9/DdnZ0DG678T6G3JfiemdZwFX+exXIF9VV7psx011yYnANFV9RERGAS+KyBBV/dEoIBKL1Xm9yr1zVvPcwh84Z2gX/vLLo2zivEjHN4Z56lTIzAy3ooDQpKFV9UMAEemkqsUtaMdNdckrgLFOu4tEJAUzCojodeMV1bXcMmM5s7/dweSf9GLqmYNs8ESk8/33cOaZkJcHL70EF10UbkUBw9WkmKoWi8gQ4ESMycTntT+5OMVioJ+I9Aa2ARcCk+odswU4FZgmIgMxt8d2udEXLrbtreCqF5awemcxt40bwNUn9bErvaKBW2+FkhITNTVqVLjVBBS3wRlXYIIzPsTMVL+PMd/bbt6vqh4RuR6Yi7kl9byqrhSRPwFLVHUW8FvgnyJyI2Y4PtkpYheR7Cmt4uJnv2R3SRXPTz6WU+wtqcinqgqSk+G556C0NGJimAOJuPGMiKwDrlLV+SJSpKrZInImcK6qXh50lY0wfPhwXbJkScjbLa3ycNE/v2BNfgn/uXIEx/T0e+fOEk5qa+G3vzX3mefNg6ToX2orIktVdXj9/W5nbjqq6nxn2ysiccA7QGiyh0cQFdW1XD5tMSu2F/P4xKOtmSOd4mKzQOTvfzf5seNj+xai24UleSLSU1U3A+uAMzArxBqvsh2D1HqVKS8uYcmmQv524TDGDIruWxwxT/0Y5quvDreioOPW0I8AQ4DNwD3A60AicFOQdEUk//x0I5+u280D5x7BhKNCm0DdcoiowgUXmJnsCI9hDiRuZ7mf89meLSLZQLKq7guasghjbX4Jj85by7ghnbjg2O7+32AJH6omgd9zz5mMnBEewxxImrX6QVUrgQQRuT/AeiKSao+Xm15bRkZKAvecM8TemopUvF644w647jpj6iFDWpWZwYWhReRSEfmriPyviCSISFsReRjYBBwddIURwN8+WMuKbcXcd+4RtE+3Ob4ikvJyM8S+5x6oqTHmboX4W8v9EHAJ8DlmaeZIYBSwFDhBVZcHXWGYWbypkCc/2cAFw7vzs8Gdwi3H0hA7dpiZ7KVL4ZFH4MYbzZC7FeLvGvpC4CRVXees3loJTFTVV4MvLfx4ar388Y0VdMlM5Y6zBoVbjqUhPB6THzsvD956y8xqt2L8GTpLVdcBqOpqESlvLWYGeGXxVtbkl/DkRUfbkquRSkICPPoodO0asal1Q4m/T6k4yQDrxi+ees9R1S3BEhdO9lXU8Oj7azmudzvGDrFD7YhC1aQHysoy95bHjw+3oojB36RYGmbyq+7RFnMvuu75D8ESFm7ue2c1e8urmXrmIDurHUlUV8Pll8Ntt5nMIpG73D8s+OuhE0OiIsJYsHYXry7ZyjWj+zKka2zEycYEu3ebGOYFC+DOO83Dftn+CH/x0LWhEhIpVFTXcvvM7+iTm8ZvxvQLtxxLHWVlJtRx61Z4+WWYODHciiISO9NTj2cWbGTb3gpenTKSlMTYXsgfVaSlwfXXw4gRMHJkuNVELNbQPuzcV8lTn2xg/BGdGNGnfbjlWMAEVQwcCKNHw69/HW41EY9NfOXDI/PWUOtVbhs7MNxSLB4P/OpXJonfc8/5P94CHIKhnWWfo0TkPOd5qoikBk9aaNm5r5I3vtnGpBE96NHeVrYIK/v2mQUijz0GN90E//pXuBVFDW5TEA0G3nKedgJmYFIQXYRZEhr1TPt8E15Vrjihd7iltG727DHD6zVr4Jln4Kqr/L/Hsh+3PfSTwD2qehgHkhrMxyQNjHrKqjy8/OVmxg7pRPd2tncOK9nZ8JOfwNy51szNwO2k2BGYulPg5NNW1VIRiYlP/3+/zqO40sMVJ/QJt5TWy/Tp5rZUr16mZ7Y0C7c99GZgmO8OERkObHDbkIiMFZE1IrJeRG5r5JjzRWSViKwUkZfdnrslqCrTv9rKkK5tOaZndiiatPji9cIf/wiTJsEDD/g/3tIkbnvoqcA7IvJ/QJKI3AJcB1zr5s1uitWJSD/gduB4VS0SkZDkxV25vZjVO4r589mDQ9GcxZfycrj0UpgxA668Ev7xj3Arinpc9dBO3uwJmOoXnwGHA+er6rsu23FTrO4q4AlVLXLaDEnFjNeWbCUpIY4JR3UNRXOWOvLzzeTXf/9rYpifeSYm0uuGG7ez3NmquhhTAaM5uClW199p6zNMMv67VPW9BrQErFhdZU0tb36zjXFDOpHZplUuWw8faWmQkmJjmAOM22vobSIyS0QuaOa9ZzfF6hKAfsDJmFthz4pI1kFvUn1GVYer6vDc3NxmSDnAx98XUFzp4ZfH2KR/IWPePFO1Ij3dBFlYMwcUt4buDXwA3Ajki8iLIjLOuTZ2g5tidXnAW6pao6o/AGswBg8ac1fuJLtNIiP72GT5QUcVHnwQxo6F++4z+2ykVMBxew2dr6r/UNWRwFCM2f6C+6Ls+4vViUgSJrXRrHrHvAmcAiAiOZgheP2a1AGjptbLh98XcOrAjrb0a7CpqoLLLjMxzBdcYDJzWoJCcz7Jmc4jAyhz8wZV9QB1xepWA6/VFasTkQnOYXOBPSKyCvgYuEVV9zRDnyu+2LiHkkqPTfwXbHbvhtNOgxdegLvuMqGPqTGzYjjicDsp1h9zXTsJY+bXgQtV9XO3DanqHGBOvX1TfbYVU4kjJNU45q3MJzUxnhP75YSiudZLaSls2WIWjlx4YbjVxDxu70MvBt4AfgV8EO2JD7xeZd6qnYzun2tjnoPF11/D0KFm5deaNaaMqyXoHEr1ycmqOjfazQywPG8v+cVVnD7YFpsLCk88AccdB48/bp5bM4eMRntoEZmoqtOdp+c3lihPVf8dDGHBZO7KfBLihFMHWEMHFI/HJLl//HE480wzEWYJKU0NuScDdYZuLOxFgagytKoyd+VORvVtbxeTBJJ9+8wM9ty5prj6gw/GfC3mSKRRQ6vqz3y2YyJMEmBdQSk/7C6zcc+BZtUq+PRTG8McZlxdQ4tIg0s+ReSLwMoJPu+t2IkInG6LtQeGbdvMz1GjTIF1a+aw4nZSbEAj+/sHSkio+Oj7AoZ2z6JD25RwS4l+/v1v6NvXrMcGaOFSXEvL8Vd98nlnM8lnu45emEUiUUNlTS0rt+/jcjvcbhl1Mcz3328KxZ0YM1dkUY+/+9DbGtlWTEnZqCpct3J7MTW1yrDuNpFBsykrg//5H5g50wyvn3gCEu3kYqTgr3LGHWCulVX1ndBICh7Ltu4FYFiPg4K4LG6ZMwfeeMNUfPzNb2yARYTR1H3o41X1M+dpiYic1NBxqrogKMqCwDdbiuiSmUJHe/186JSVmRjmX/4SBg2CwTbDSyTSVA/9HAcmw/7TyDEKtCzLQAhZtnUvQ23vfOjMnAnXXAPvvQdHH23NHME0OsutqgN8trs38ogaMxeUVJJXVGGvnw8FVZO47xe/MLPZXW2apkinWbWtROREwKOqiwKsJ2gs22Kun20P7ZKqKpgyxdyamjgRnn/epAyyRDRuF5bMd0yMiNwMzARmisitwRQXSL7N20d8nDCki6337Ir/+z9j5rvvhv/8x5o5SjiURPt1vfHVmLxfJcCnwIOBlxV41uaX0Kt9G1KT7PriJvF6IS7OlG494ggYMybciiyHgNuVYnGAV0T6AAmqulJVtwBRk4xr/a5SDuuQHm4Zkc3cuSaGOT/f3Fu2Zo463Br6c+BvwEOYRAc45g5aiqBAUu3xsnlPOf06ZIRbSuTyxBMwfry5r1xdHW41lmbi1tCTgUpMcsA7nX2DgMeCoCngbNpTRq1XbQ/dEB4P3HCDGWKfcQYsXAjdbVrjaMXVNbSq7gJ+V2/fbGB2MEQFmnX5pQDW0A1x110mIcHNN5tbVDaGOapxO8udICJ3iMhaESlzft4hIq4X8bopVuccd56IqFMMLyCsLyhFBPrmWkMfxE03mUycDz9szRwDuB1yPwicAfwGU6fqN8A4wFW5QJ9ideMwQ/WJIjKogeMyMIkIv3SpyxXrCkrolp1qZ7jr+PRT+PnPzb3mdu3MfWZLTODW0OcDZ6rqHGeGew6m2JzbvKxuitUB/Bkz8Vbp8ryuWF9QaifE6njhBTj1VJNhZNeucKuxBBi3ho4HvPX2eWm4ZlVDNFSs7kfrCEVkGNDduTZvFBGZIiJLRGTJLhcfSE+tl427y+jX2q+fvV64/XaYPNnEL3/xBXTrFm5VlgDj1tAzgFkicqqI9BORMZjbV/91+f4mi9WJSBzwV+C3/k50qMXqthZVUO3x0re1G/qmm8yk15QpJsgi265pj0XcrhS7BXO76jmgM6am1SvA3S7f769YXQYwBJjvpAvuhPkCmaCqS1y20XDDReUA9GzXpiWniX6mTDEBFtdfb2OYYxi3t62qgN87j+awv1gdJvPJhZiyOnXn3wfsr0kjIvOBm1tqZoDKGnOl0CapWXEo0c3SpSb08Z57TAzzoIPmIS0xRpNDbmd4vUBECkXkAxFpVriky2J1QaHKYwp9JCe2sgqTM2eaa+WXXjIF4yytAn+f8scxPepkYDdm+WezcGbI+6tqX1W919k3VVXrl5VFVU8ORO8MUOX00MkJrcTQqiZ53y9+AUcdBV99ZbNxtiL8jUOPwcw8V4jIx8D3IdAUUKo8dYZuJfegr7/ehD7aGOZWiT9DJ6lqBYCqlohI1BX23T/kbi099Nix0KmTSbNrJ79aHf4MnSwiU32ep9Z7jqr+KfCyAsf+HjqWr6FXrTITYJdcAmedZR6WVok/Q78G9PN5PqPecyXCqbuGToqPUUPPnQvnnw9t28K555rMnJZWi7+83JeESkiwqPLUkhAnJMSioR9/HH79a5NZ5O23rZktrleKRS1VHm/sXT+rmhjmG24wdZhtDLPFIcY+6QdT5aklOTHGZrhFoEsXuOUWc785vZUva7XsJ+aXT1XVxFAPvWED7NgBJ5wAt91mZ7EtBxH7ho6VIfeCBWbSKzsbVq+GhJj/11magetPuoicIiJPi8ibzvOjRWR08KQFhipPbfQvKpk2zWTgzMmBd9+1ZrY0itsURP+LibTaCpzi7K4G7g2SroBR5fFG7z1or9cMrS+7DEaPhkWL4LDDwq3KEsG4/aT/FhijqvdwINHBamBgUFQFkKi+hhaBzZtNobg5c2wMs8UvbsduGcBmZ7tuMUkCppeOaKo8tdEXOpmXBzU10Ls3vPiiSd5nJ8AsLnDbdS0Ebq637zrgk8DKCTxRNym2ZAkcd5wJrlA118vWzBaXuP2k3wBcKCLrgQwRWQlcAtwYNGUBIqquoWfMgJNOgqQkePZZa2TLIeM2Y8k2ETkGGIUp8L4VWKSqtcEUFwiiYpZbFe67z0RI/eQn8MYb0KFDuFVZohDXF5eq6gU+cx5RQ1RMilVVGRNfdJHpmW0Ms6WZuDK0iPxAI5FVqtonoIoCTERfQ+/aZcybkQEffmgipuww29IC3PbQV9Z73hlzXT09sHICT8Su5V650gRWDB8Or78OmbYQvaXluL2G/rD+PhH5EJhDC/KMBRtVjcwe+r33TAxzWhr87nf+j7dYXNKST3oF4Hq47a9YnYjcJCKrRORbEflQRHq2QBsANbWKagSlH1KFxx4zZVv79jUJ/I49NtyqLDGE22voqfV2tcEUr5vn8v11xepOwyTdXywis1R1lc9h3wDDVbVcRK7F1Li6wM35G6O6NsISBBYVwb33mhRBL71kwx4tAcftNXS/es/LMAad5vL9+4vVAYhIXbG6/YZW1Y99jv8CuNjluRulqiZCcnIXFxvztmtn1mP37AlxETJqsMQUfg3t9K7vY5LjN7cqZEPF6kY0cfwVwLuN6JkCTAHo0aPpvP8HUviG0TwbNpjJr3PPNb1z797h02KJefx+0p3FI4+1wMzgp1jdjw4UuRgYDjzciB7XxerCnpN7wQIYMQIKCuBnPwuPBkurwm3X9Y6IjG9BO/6K1QHgVLX8AzDBqafVIsKak7suhjk310x+nXRS6DVYWh1ur6HjgJkishAzdN7fu6rq5S7e32SxOthfH/ppYKyqFrjU1ST7y+CE+hp682a4+mo4+WR47TXIygpt+5ZWi1tDr6ORIbAbVNUjInXF6uKB5+uK1QFLnPpWDwPpwOtOSdktqtqiQnYhH3J7PCY6qmdP+OQTOOYYSEwMTdsWC34MLSITVXW6qt7R0oZUdQ5mIYrvvqk+22Na2kZ9QjrkzsuDCRPg5pth0iQYOTL4bVos9fD3SX86JCqCxIHKk0HuoRcvNgtE1q+3WUUsYcWfoaM6UiAkda1ee81MeKWkwOefw7hxwWvLYvGDv2voeBE5hSaMraofBVZS4Aj6kHv5crjgAhvDbIkY/FafxGT7bMzQyiGs5w41QZ8UO+ooeOUVOPtsG8NsiQj8dV1lqtpHVXs38ohYM4PP0s9A9tAFBXDaaSb3F5ge2prZEiHE9ILigF9Dr1hhEvgtXAjbtgXmnBZLAGkVk2IBqQ09Z465Vq6uNks6zz675ee0WAJMk590Vc0IlZBgELDa0PPnm5BHG8NsiXBie8gdqASBJ5wAf/4zfPopdOvW8vNZLEEitg3t8TY/n1hREVx6qSnfmpAAv/+9TUhgiXhi3NC1zeuh16+HUaNg+nSzCsxiiRKirOjToVHl8ZJ0qIb+5BOTjADggw9s2KMlqojtHvpQr6HnzDH3mDt0sDHMlqgktg19qGVwRo0ytZgXLTIz2hZLlBHjhnbRQ5eWmppSlZUmUurpp21CAkvUEvuGbmqV2Nat5pbU/febxSIWS5QT44ZuYsi9eLFZxrlxI7zzDpx+emjFWSxBILYN3dik2NtvH4hhXrQIxo4NvTiLJQjEtqEbu4bu1w9OPdXMZA8eHHphFkuQiHFD+wy5KytN7WVVGDAAZs82KXYtlhgiZIZ2UawuWURedV7/UkR6tbTN/ZNi+fnw05/CVVfBl1+29LQWS8QSEkP7FKsbBwwCJorIoHqHXQEUqephwF+BB1vablWNl255683k17Jlpg6zzcZpiWFC1UPvL1anqtVAXbE6X84GXnC2ZwCnipOguzmoKiPXfMnkWy6CmhpzW+q885p7OoslKgiVoRsqVte1sWNU1QPsA9o3t0GP19SGLurhxDAPH97cU1ksUUOoDO2mWJ2rgnYiMkVElojIkl27djXaYJwIVz9wPdULP7cxzJZWQ6gM7aZY3f5jRCQByAQK65/IbfXJ+DhhVN/2dM+xMcyW1kOoDL2/WJ2IJGGK1c2qd8ws4FJn+zzgI1VtsOSsxWJpmJDEQ7ssVvcc8KKIrMf0zBeGQpvFEkuELMGBi2J1lcAvQ6XHYolFYnqlmMXS2rCGtlhiCInmeScR2QVs9nNYDrA7BHL8ESk6wGppiEjRAe609FTVg27zRLWh3SAiS1Q17KtKIkUHWC2RrANapsUOuS2WGMIa2mKJIVqDoZ8JtwCHSNEBVktDRIoOaIGWmL+GtlhaE62hh7ZYWg3W0BZLDBEThg5HeqMWaLlJRFaJyLci8i86UC8AAAdRSURBVKGI9AyXFp/jzhMRFZGg3LZxo0NEznf+LitF5OVg6HCjRUR6iMjHIvKN8z8aHyQdz4tIgYisaOR1EZF/ODq/FZGjXZ1YVaP6gQn22AD0AZKA5cCgesf8L/CUs30h8GoYtZwCtHG2rw2nFue4DGAB8AUwPEx/k37AN0C287xDGP8/zwDXOtuDgE1B0nIScDSwopHXxwPvYvIEjAS+dHPeWOihQ57eqCVaVPVjVS13nn6BiQ0PBm7+LgB/Bh4CKsOo4yrgCVUtAlDVgjBqUaCts53JwXH7AUFVF9BAvL8PZwP/VsMXQJaIdPZ33lgwdMjTG7VQiy9XYL6Fg4FfLSIyDOiuqrODpMGVDqA/0F9EPhORL0QkWJUP3Gi5C7hYRPIw0YE3BEmLPw71swTERn3ogKU3CpEWc6DIxcBwYHQQdPjVIiJxmOyqk4PUvisdDgmYYffJmBHLpyIyRFX3hkHLRGCaqj4iIqMwMfpDVNUbYC3+aNZnNhZ66IClNwqRFkRkDPAHYIKqVgVBhxstGcAQYL6IbMJcp80KwsSY2//PW6pao6o/AGswBg80brRcAbwGoKqLgBRMsESocfVZOohgXPCH8oH5dt8I9ObARMfgesdcx48nxV4Lo5ZhmImZfuH+u9Q7fj7BmRRz8zcZC7zgbOdghprtw6TlXWCysz3QMZEE6X/Ui8Ynxc7gx5NiX7k6ZzA/VKF6YGYE1zpG+YOz70+YHhDMt+zrwHrgK6BPGLV8AOQDy5zHrHBpqXdsUAzt8m8iwKPAKuA74MIw/n8GAZ85Zl8GnB4kHdOBHUANpje+ArgGuMbnb/KEo/M7t/8bu/TTYokhYuEa2mKxOFhDWywxhDW0xRJDWENbLDGENbTFEkNYQ0cYIvKSiNwVbh3+cCKWTmzi9XkiclEoNVmsoYOGiGwSkQoRKfV5dAmTlpdEpNrRUOiYrX9Lzqmqh6vqp8757xGRafVeP11V/9OSNuojIglOmGeZ87vkicjDzjJWN+8f46yKi1msoYPLWaqa7vMISuSOS+5T1XTMcsJC4Pkwamkpg53f5afAJRwoctjqsYYOMSISJyIzRGSniOwVkfkiMrCRYzuIyBznuEIRWeDzWjcReUNEdonIDyJynZv2VbUMs0ppiHOeFCeQfoeIbBORR8VUCPXXfp6InCwiZwK/Ay5yes2lzusLRWSyiKSKSLGIDPB5bydn9NLeeT5BRJY77SwUkSEuf5e1wOfAUJ9zXykiq0WkREQ2iMiVzv5M4G2gh8+IqYPz//i9c+xuEXlFRLLdtB+JWEOHh9mY4INOwArgxUaOuwWz9jjXOfYOABGJd86xGBNSdxpwi4ic6q9hEckAJmESCgBMxUR9HYlZZ348cHtT7fuiJvTyIeA/zijkmHqvVwBvYqKY6rgA+FBV94jIscA/gSsxIa3PA2/Vfan4+V0GOnrX++zOx6yDbouJs35MRI5U1X3AWcAWnxFTAXCTc/xJmACIMuAf/tqOWIK1Zra1P4BNQCmw13m82chxOZiwuDTn+UvAXc72fcBMoG+99xwPbKy37w7gn4208RImgcFezPrhN4Hezmub8VmvjPlwr2+qfee1POBkZ/seTMih7+sLORDkMBZY6/Pal8AkZ/ufwJ313rsBOL6BNhOcv1Uxxnjq/G5JTfwfZgPXOdtjqJeBBFgHjPZ53h2oAuLC/RlqzsP20MHlHFXNch7ngOldReQhEdkoIsUc6F0aCtF7AGO4D50h4S3O/p6YoePeugdm2NupCS0PODo6q+o5asIUATrz4/pgmzkQSN9Y+4fKB5iMG8eISF9gMPCWz+9ya73fpTNNB/MfiQn/nASMAtrUvSAiZ4rJG1fonOt0mg5/7AG87dP2d5gvig7N+k3DjDV06PkfTMTPTzFx2Yc5+w8KaFfVYlW9UVV7AedgPvijMeGF63y+LLJUNUNVz2qGnh0YU9XRA9jmp/2DpDbVgJosMa9jht2TMLHPZc7LW4G76/0ubVT1NT/n9KrqdGAJ8EcAEUnFpJi6H+ioqlnAPA78bRvSmQecVq/9FFXd2VT7kYo1dOjJwAzp9mB6lnsbO1BEzhKRviIimLRJtc5jEVAtIr91JrXiReQIETmmsXM1wXRgqojkiEguZuj+kp/265MP9HKOa4yXMdfOk5ztOp4BrhORY8WQ7rSb5lL//cA1jvZkTJzzLqDWmbDznVfIB3KceYQ6ngLuE5Eezu/cQUQmuGw74rCGDj3/wgTNbwdWYmZpG+Nw4CPMtfhnwN9VdaHT443HJL3bhCk9+jQHktsdCndjYn+/A77FXN/e31T7DZzjVYyRCkXkq0ba+RzwYCbY5tXtVNUvMdlPnwSKMLHKF7sVr6rLMF9wN6tJWXQj8Abm1tx5mGvoumNXAP8FNjlD7A6YOOz3MJcVJY7OY922H2nYeGiLJYawPbTFEkNYQ1ssMYQ1tMUSQ1hDWywxhDW0xRJDWENbLDGENbTFEkNYQ1ssMcT/A/M13tdVbasJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 252x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(3.5, 3))\n",
    "fpr = mdl_res[0]['mean_fpr_minute']\n",
    "tpr = mdl_res[0]['mean_tpr_minute']\n",
    "plt.plot(fpr, tpr)\n",
    "\n",
    "plt.plot([0, 1], [0, 1], 'r--', label='Chance')\n",
    "plt.xlabel('False Positive Rate', size=12)\n",
    "plt.ylabel('True Positive Rate', size=12)\n",
    "# plt.title('ROC for Minute-wise Prediction', size=12)\n",
    "# plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.savefig('../archive/Model_Minute_ROC_LogReg.png', dpi=600)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-06T21:48:32.671970Z",
     "start_time": "2020-02-06T21:48:32.665994Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.61"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mdl_res[0]['thres_best']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Testing performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T03:44:14.233421Z",
     "start_time": "2020-02-09T03:44:13.965169Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=0.05, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=10000.0,\n",
       "                   multi_class='auto', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Model fitting using the whole training set\n",
    "scaler = preprocessing.StandardScaler().fit(df_train[feature_col])\n",
    "mdl=LogisticRegression(C=0.05, max_iter=1e4)\n",
    "mdl.fit(scaler.transform(df_train[feature_col]), df_train['apn'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-07T20:18:41.668257Z",
     "start_time": "2020-02-07T20:18:41.650302Z"
    }
   },
   "outputs": [],
   "source": [
    "res = {'mdl': mdl, 'scaler': scaler}\n",
    "with open('../resources/model_logreg.pkl', 'wb') as f:\n",
    "    pickle.dump(res, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Group-wise performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-08T23:13:34.281352Z",
     "start_time": "2020-02-08T23:13:34.140759Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           A       0.83      0.71      0.77         7\n",
      "           B       0.67      0.67      0.67         3\n",
      "           C       0.80      1.00      0.89         4\n",
      "\n",
      "    accuracy                           0.79        14\n",
      "   macro avg       0.77      0.79      0.77        14\n",
      "weighted avg       0.79      0.79      0.78        14\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARgAAADvCAYAAAApbsuZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAd9klEQVR4nO3debxd093H8c83iRCEEBIJkdTUmIdWKTU8ah4eUUPK06oagtejOlBVlBgiNbWUqlKlZqqPmocWqSFmLTVVDRlUEglCRAxJfs8fa53YOU5uTnLvvucO3/frdV737LX28Nv73vO7a6999tqKCMzMytCl0QGYWcflBGNmpXGCMbPSOMGYWWmcYMysNN0aHUBn0GPDIzrdpbpHbx7Z6BCsFa2/ck/VKncLxsxK4wRjZqVxgjGz0jjBmFlpnGDMrDROMGZWGicYMyuNE4yZlcYJxsxK4wRjZqVxgjGz0jjBmFlpnGDMrDROMGZWGicYMyuNE4yZlcYJxsxK4wRjZqVxgjGz0jjBmFlpnGDMrDROMGZWGicYMyuNE4yZlcYJxsxK4wRjZqXxo2M7sJduP5lp0z9m1uzZzJw1m6/9z5mNDqlUF559Mk8/9hBL91qGcy65odHhtJq2vN/tpgUj6WuSRkt6T9I7kh6WtHGj42rrdhx2Hpt+8+cdPrkAbL39bhx3+vmNDqPVteX9bhcJRtJSwG3A+cCywIrAycDHrRiDW3tt3FrrbcSSPZdqdBitri3vd7tIMMAaABFxbUTMiogZEXFPRDwLIOlASS9KelfS3ZIG5vKLJJ1dXJGkmyX9KL/vL+lPkiZLel3SkYX5hku6UdJVkt4HDpDURdKxkl6V9LakGyQt22pHYQFFBLdeeAQPX30MB35j80aHY51Qe0kwLwOzJP1B0k6SlqlUSBoCHAd8A1geeBC4NldfAwyVpDzvMsD2wHWSugC3As+QWkRfB34gaYfCdncHbgR6AVcDRwJDgK2A/sC7wK9L2eMWsM13f8lm+53BkCMu5NChW7D5Rqs2OiTrZNpFgomI94GvAQFcAkyWdIukvsChwMiIeDEiZgKnAxvkVsyDeZkt8qr2Ah6JiDeBjYHlI+KUiPgkIl7L6/5mYdOPRMSfI2J2RMzI2zo+It6IiI+B4cBetU6fJA2T9KSkJ2dOeb7Fj0k9Jkx+D4DJ737ALfc9y8ZrD2pIHNZ5tYsEA5ATyAERsRKwDqkFcS4wEDhP0lRJU4F3AAErRkQA1wH75tXsR2qJkJfrX1kuL3sc0Lew2fFVYQwEbirM/yIwq2qZSrwXR8SXI+LL3ZZbu/kHYAEtvlh3llx80Tnvt/3qYJ5/9c1Wj8M6t3bZcRkRL0m6nNSiGA+MiIir5zH7tcA9kn4ObALskcvHA69HxOpNbapqejxwYEQ8vNDBt5I+vXty/S8OAaBb165cf+eT/GX0iw2OqlznjjiOF559imnvTeWwfXdmn/2Hsc1OQxodVuna8n4r/ZNv2yQNBnYBro+INyQNILVMXgDuAE4FhkbE85KWBraPiD8Wln8B+A/wQUTskcu6Ao8DNwC/Aj4B1gR6RMQTkoYDq0XEtwrr+SGpX+Y7ETFW0vLAZhFxc1Px99jwiLZ/kFvYozePbHQI1orWX7mnapW3l1OkaaTWx2OSpgOPAs8BR0XETcAZpI7b93P5TlXLXwtsS+r0BSAiZgG7ARsArwNTgN8BSzcRx3nALaQW0bQcxybN3juzDqpdtGDaO7dgrKNr7y0YM2uHnGDMrDROMGZWGicYMyuNE4yZlcYJxsxK4wRjZqVxgjGz0jjBmFlpnGDMrDROMGZWGicYMyuNE4yZlcYJxsxK4wRjZqVxgjGz0jjBmFlpnGDMrDROMGZWGicYMyuNE4yZlcYJxsxK4wRjZqWZ56NjJe1c70oi4o6WCcfMOpJ5PnhN0uw61xER0bXlQup4nhk3rdM9eO2h8W83OoSGOGiTQY0OoSEW60bNB6/NswUD9CgpFjPrJOaZYCLi49YMxMw6nro7eSVtI+lGSX+XtFIuO0DSVuWFZ2btWV0JRtLewK3AZGAw0D1XLQ4cW05oZtbe1duCOR44LCIOB2YWykcDG7Z4VGbWIdSbYNYAHqhR/j7Qq+XCMbOOpN4EMxFYrUb55sBrLReOmXUk9SaYS4FzJX0JCKCvpKHAWcDFZQVnZu1bU9+DKTodWJbU57II8DCpL+a8iDi3pNjMrJ2rK8FE+rrvUZJOAdYltXz+GRHvlhmcmbVv9bZgKqaT+mMAprVwLGbWwdT7PZhFJP0cmAr8K7+mSjpDUvemlzazzqreFswFwH8D3wceyWVfBU4lXaY+tOVDM7P2rt4E801gaETcVSh7QdKbwHU4wZhZDfVepv4IGFujfAzwSYtFY2YdSr0J5jfAccX+FkmLkO5D+k0ZgZlZ+9fUiHY3VBXtCGwv6e95egPSmDF3lxSbmbVzTfXBzKqavr1q+v4WjsXMOpimBpzatzUDMbOOx08VMLPS1P1NXkn7AvsCK/PZgFMARMRaLRyXmXUA9X6T9wfARcCrpBHt7gPGA/2BG0uLzszatXpPkQ4HhkXED4FPgV9ExA7Ar4DlywrOzNq3ehPMAODR/H4G0DO/vxLYp6WDMrOOod4EM4k0HgzAOOAr+f1AqP3AJTOzehPM/cCu+f0fSKPb3QncANxcRmBm1v7VexXpsMq8EXG+pPdJ4/HeC5xfUmxm1s7VO6LdJxRuaoyIP5BaMg0jaRDwOrBIRMxseu7O58KzT+bpxx5i6V7LcM4l1Xd9dEzT3nmLe353Fh++9y6SWGerndlguz0aHVbpHn7wAc74+Qhmz5rNHnvuzUGHDGt0SHM0dS9S3d9tiYgX5jePpDGky9r9I2JKofwfwPrAFyJiTL3bLFuO9+CI+GujY1kYW2+/GzvuPpRfn3lio0NpNV26dGWLocPoM3B1PpnxIdedcgQD1tqI3isObHRopZk1axanjziF315yGX379mW/oXux9X9tw6qr1XoISOtrqgXzHOkJArUo11V+dq1ze6+Tvqx3PoCkdUk3TLYqSd06eqtnrfU24q2JbzY6jFa1RK/eLNGrNwDdeyzOMv0GMH3qlA6dYJ7757MMGDCQlQYMAGDHnXdh1P33tpkE01Qn75rAWvN4rVn1s15XAvsXpr8DXFGZkLS0pCskTZY0VtIJkrrkuq6SzpY0RdJrwC7FFedlL5U0QdJ/JJ0mqWuuO0DSw5J+KekdYLikVSXdJ+ntvM6rJfXK819J+sbyrZI+kHRMLt9U0mhJUyU9I2nrBdh3a0XvT5nI5HGv0neVwY0OpVRvTZrECv1WmDPdp29fJk2a1MCI5jbPBBMR/6r3tQDbexRYStKa+cM/FLiqUH8+sDSwCrAVKRl9N9cdQrqStSHwZWCvqnX/gfQoldXyPNsDBxfqNyE9JK4PMILU+hpJOm1bk/Rdn+F5379Nuhy/W0QsGRFnSlqRdEf5aaRL9kcDf5JU84uGkoZJelLSkzdec1m9x8dawCcfzeD2X5/KlvsexqI9lmh0OKWKGicZUtv55siCPlWgJVRaMX8DXgL+k8srCWfDiJgGTJN0DvBt0oPf9gHOjYjxAJJGAlvn932BnYBeETEDmC7pl8Aw4Ld5/W9GROWK10zglfwCmCzpF8BJTcT9LeCOiLgjT/9F0pPAztTo8I6Ii8kPpXtm3LR5nWpaC5s1cyZ3/PpUvrjpNqz2pa81OpzS9e27AhMnTJwz/dakSfTp06eBEc2tUQnmAeALFE6PgOVIN1EWh+YcC6yY3/cn3f9UrKsYSHog3IRC9u5SNX/xPZL6kG512IL0zeQuQFPPeRoI7C1pt0LZInhcnDYjIrj3sl+wbL8BbLTDno0Op1Wsvc66jBs3hjfeGE/fPn25647bGXnWOY0Oa45WTzARMVbS66T//AcVqqaQ7nMaCFSuSq3MZy2cCaTTGAp1FeOBj4Hlmui8rW5FjMxl60XE25KGkJ6eMK/5xwNXRsQh89q3tuTcEcfxwrNPMe29qRy2787ss/8wttlpSKPDKtWEfz/PS4/cS++VvsA1Jx0OwGZ7fpdB631lPku2X926deOnx5/I4cMOZvbsWQzZY09WW231Roc1RyNaMJASyzIRMV1SJYZZpG8Gj5C0P6mf40fA2bn+BuBISbeRHgB3bGVlETFB0j3AOZJ+BnxAaiGtFBF/m0cMPYH3SM93WhH4cVX9JFJfUMVVwBOSdgD+Smq9bAq8EhFvLPARKNkPjj+90SG0uv5rrMORv+98I7huseVWbLHlVo0Oo6YFGnBK0pKS1s8Dfi+0iHg1Ip6sUfU9UvJ4DXgIuAb4fa67hDT+7zPA08D/VS27P+kU6wXSqc6NQL8mwjgZ2IiUZG6vsb6RwAn5itHRue9nd+A4YDKpRfNjPGiX2TwpPXZ6PjNJS5CeHvAtYDawRkS8JukCYEJEjCg3zPatM3byPjT+7UaH0BAHbTKo0SE0xGLdat/0XO9/35GkgaY2Iz0jqeIeYO/mhWZmHVW9fTC7A/tExGOSiv+NX2DufgozsznqbcEsD7xVo7xjf4vJzJql3gTzFOmyckWlFXMg8EiLRmRmHUa9p0jHA3dIGpyX+V9Ja5O+Sds2r4+ZWcPV1YKJiAdIiaQP6Ytv3yBdTt48Ih4vLzwza8/q/qJdRDxFulfIzKwudSUYSYs3VR8RH7ZMOGbWkdTbgvmAeQ8+BfUPOGVmnUi9CWanqulFSGOuHAz8rEUjMrMOo95Bv2vdQXabpJdJtw9cUaPezDq55t6o9ySwTUsEYmYdz0InGEndgf/ls/FazMzmUu9VpMnM3ckroBfpWUn711zIzDq9ejt5T6iank0aE2V0RNS6R8nMbP4JJo849ylpwOuJ85vfzKxivn0weYzbC4BFyw/HzDqSejt5Hyc93tXMrG719sFcQBpQuz9p6Ibpxcp6nk1tZp1PvQnmhvzzwvyzckVpQZ9NbWadSL0JZs1SozCzDqnJBCPp98D3F/D502ZmwPw7eb8D9GiNQMys45lfgqn5rBMzs3o0+eA1SbOBvhExufVC6ng+mtnkWDrWgQw6/MZGh9AQEy/Zq2ZjpJ5O3olS0w2ZiPBVJDP7nHoSzDBgatmBmFnHU0+CudU3NJrZwphfJ6/7DsxsofkqkpmVpslTpIho7pCaZtaJOYGYWWmcYMysNE4wZlYaJxgzK40TjJmVxgnGzErjBGNmpXGCMbPSOMGYWWmcYMysNE4wZlYaJxgzK40TjJmVxgnGzErjBGNmpXGCMbPSOMGYWWmcYMysNE4wZlYaJxgzK40TTDNIGi7pqkbHMS8PP/gA/73LDuy643ZcesnFjQ6nVXTGfQboIvjLz77Old/bvNGhzKXNJxhJYyRNkrREoexgSaMaGFabN2vWLE4fcQoXXvQ7brrldu664zZefeWVRodVqs64zxWHbLs6/54wrdFhfE6bTzBZN+D7zVmBkvayv8323D+fZcCAgaw0YACLdO/Ojjvvwqj77210WKXqjPsM0G+ZHmy7bj+ufuj1RofyOe3lA3cWcLSkXtUVkjaT9ISk9/LPzQp1oySNkPQw8CGwSi47TdJoSR9IulVSb0lXS3o/r2NQYR3nSRqf656StEUr7G+zvTVpEiv0W2HOdJ++fZk0aVIDIypfZ9xngFOHrs+pNz5LzG50JJ/XXhLMk8Ao4OhioaRlgduBXwG9gV8At0vqXZjt28AwoCcwNpd9M5evCKwKPAJcBiwLvAicVFj+CWCDXHcN8EdJi7XcrpUjajz1V+rYD+rsjPu83Xr9mPL+xzw7bmqjQ6mpvSQYgBOB70lavlC2C/DviLgyImZGxLXAS8BuhXkuj4jnc/2nueyyiHg1It4D7gRejYi/RsRM4I/AhpWFI+KqiHg7L38OsCjwxfkFK2mYpCclPdmIzsa+fVdg4oSJc6bfmjSJPn36tHocrakz7vPGq/Zm+w368cTInbho2CZs/sXlueCgjRsd1hxNPjq2LYmI5yTdBhxLamUA9OezVknFWFLLpGJ8jdUV280zakwvWZmQdBRwcN5WAEsBy9UR78XAxQAfzazxr7Vka6+zLuPGjeGNN8bTt09f7rrjdkaedU5rh9GqOuM+n37Tc5x+03MAbLbG8hy+wxoccekTDY7qM+0mwWQnAU8Dlb+aN4GBVfOsDNxVmF7oD3fub/kJ8HXg+YiYLeldoM23u7t168ZPjz+Rw4cdzOzZsxiyx56sttrqjQ6rVJ1xn9u6dpVgIuIVSdcDRwL/BO4Azpe0H3ADsCewFnBbC22yJzATmAx0k3QsqQXTLmyx5VZsseVWjQ6jVXXGfa4Y/fJkRr88udFhzKU99cFUnAIsARARbwO7AkcBbwPHALtGxJQW2tbdpD6al0mnXh9R+5TLzGpQRKt3D3Q6jeiDscYYdPiNjQ6hISZeslfNboP22IIxs3bCCcbMSuMEY2alcYIxs9I4wZhZaZxgzKw0TjBmVhonGDMrjROMmZXGCcbMSuMEY2alcYIxs9I4wZhZaZxgzKw0TjBmVhonGDMrjROMmZXGCcbMSuMEY2alcYIxs9I4wZhZaZxgzKw0TjBmVhonGDMrjROMmZXGT3bs4CQNi4iLGx1Ha+qM+wxtc7/dgun4hjU6gAbojPsMbXC/nWDMrDROMGZWGieYjq9NnZO3ks64z9AG99udvGZWGrdgzKw0TjBmVhonGGuXJA2SFJK6NTqWjkbScElXtcS6nGDaIElfkzRa0nuS3pH0sKSNGx1Xc0gaI+kTSctVlf8jJ4pBjYmsthzvtiWvf5KkJQplB0saVdY2G8EJpo2RtBRwG3A+sCywInAy8HErxlBWq+B1YN/CdtYFepS0rXlqQ62ebsD3m7MCJW32c9xmA+vE1gCIiGsjYlZEzIiIeyLiWQBJB0p6UdK7ku6WNDCXXyTp7OKKJN0s6Uf5fX9Jf5I0WdLrko4szDdc0o2SrpL0PnCApC6SjpX0qqS3Jd0gadlm7tuVwP6F6e8AVxTiWFrSFTnGsZJOqHx4JHWVdLakKZJeA3ap2telJV0qaYKk/0g6TVLXXHdAbgX+UtI7wHBJq0q6L+/bFElXS+qV578SWBm4VdIHko7J5ZvmluVUSc9I2rqZx+Ms4OjKdqv2ZzNJT+RW7BOSNivUjZI0QtLDwIfAKrnstBzfB5JuldQ779f7eR2DCus4T9L4XPeUpC2auS+1RYRfbegFLAW8DfwB2AlYplA3BHgFWJP03+8EYHSu2xIYz2dfPVgGmAH0J/0jeQo4EegOrAK8BuyQ5x0OfJrX34XUqvgB8CiwErAo8Fvg2mbs1xhgW+BfOf6uOd6BQACDSMnmZqBnnn4ZOCgvfxjwEjCA1LK7Py/XLdf/Oce4BNAHeBw4NNcdAMwEvpePWw9gNWC7vG/LAw8A51bHW5heMf9eds7HaLs8vXwzj8f/AaflsoOBUXn/3gW+nePdN0/3zvONAsYBa+f6RXLZK8CqwNLAC/n4bZvnuQK4rLD9bwG9c91RwERgscLfw1Ut8vfc6A+UXzX/+NYELgfeyB+MW4C+wJ2VD1yerwvpP9hAQPmPbstcdwhwX36/CTCuahs/rfzB5T+oB6rqXwS+XpjuR0pC3RZynyofqBOAkcCOwF/yH3jkD8bHwFqFZQ4FRuX39wGHFeq2z8t1y8fmY6BHoX5f4P78/oDq/a8R3xDg79XxFqZ/AlxZtczdwHeaeTzWAd4jJblKgvk28HjV/I8AB+T3o4BTqupHAccXps8B7ixM7wb8o4l43gXWL/w9tEiCaSvnolYQES+SPhRIGgxcBZxLSiTnSTqnMLuAFSNirKTrSB+sB4D98nLk5fpLmlpYrivwYGF6fFUYA4GbJM0ulM0ifZj/s/B7x5U5vi9QOD0CliO1rsYWysaSWg6QWmLjq+qKsS4CTJBUKetSNf9c+yepD/ArYAtSi6kL6UM2LwOBvSXtVihbhNSSWmgR8Zyk24BjSUkd0r6OrZq1eCzg878vgEmF9zNqTC9ZmZB0FCmh9Scl6qVIv4MW5T6YNi4iXiK1ZtYh/VEdGhG9Cq8eETE6z34tsFful9kE+FMuHw+8XrVcz4jYubipqk2PB3aqWmaxiGhOciEixpI6e3cmnR5UTCG1kAYWylbms2Q2gXR6VKwrxvoxsFwh1qUiYu3ipqtCGZnL1ouIpUinDGpi/vGkFkzxeCwRET+fzy7X4yRSi7OSQN5k7uMAcx+LWvHVLfe3/ATYh3QK3ovUilKTCy4EJ5g2RtJgSUdJWilPDyC1Sh4FLgJ+KmntXLe0pL0ry0bE34HJwO+AuyOi0mJ5HHhf0k8k9cgdpuuo6UvfFwEjCp3Iy0vavYV28yBgm4iYXiibBdyQt9kzb/dHfNYKuwE4UtJKkpYh/ccHICImAPcA50haKndQryppqyZi6Al8AEyVtCLw46r6SaS+qoqrgN0k7ZCP32KStq78npojIl4BrgcqHe93AGtI2k9SN0lDgbVIVxdbQk/SqfdkoJukE0ktmBbnBNP2TCO1Ph6TNJ2UWJ4DjoqIm4AzgOvy1Z7nSB3BRdeSzu2vqRRExCzSOfgGpNbDFFISWrqJOM4j9f3cI2lajmOTZu9diufViHiyRtX3gOmkDuiH8j78PtddQurzeAZ4mrlbP5CuTnUndW6+C9xI6jeal5OBjUj/uW+vsb6RwAn5itHRETEe2B04jvTBHE9KSi31GTqF1EFNRLwN7ErqfH0bOAbYNSKmtNC27ib1571MOvX6iNqnXM3mmx3NrDRuwZhZaZxgzKw0TjBmVhonGDMrjROMmZXGCcbMSuMEYw0h6TlJwwvTYyQd3YA4vqz5jEeT71S+YAHWuXVeZ7O+ei/p8nwbQbvlBGPAnD/myK9PJb2Wh0dYYv5Lt4iNgQvrmTEPv/BByfFYC/DNjlb0V9KdvIuQbgL8HenbpYfXmlnSIhHxaUtsOCImt8R6rG1xC8aKPo6IiRExPiKuAa4mDWNQbPbvLOlxSZ8AO+S63fKgRR8pDWY1QlL3ykol9VEa/GqG0kBSB1ZvuPoUKd9T9BulAaQ+Uhpka2ge5OkyYIlCi2t4Xqa7pDMkvSFpeh5kaYeq7ewo6aW8zgfJA3wtCEnfyuueJuktSX/M9zNV21RpSNCP8vH5UtV6NpP0N0kfKg2S9RulEQ07DCcYa8oMUmum6AzSmC6DSfdL7UBKRBeQBkA6ENgLOL2wzOWkAZ62JSWs/UkDStWkNObCncBWwHdJN/r9CPgEGE0aDOtD0r1G/YDKSH6X5WX2A9YlDdp1q6T183oHkAam+gvpvqzzgTPrPRgF3Ul3QK9PumdoOdI9YNXOJt21/GXS/VW3S1o8x7Iu6QbNW/J6vpFj+n2N9bRfLTGojF/t/0VKArcVpr9Cuiny+jy9NWmIgD2rlnsA+FlV2RDSncoitRAC2LxQP5B09/TwQtkY4Oj8fjtgNrDmPGI9APigqmzVvMzKVeV/Bi7M708n3eCnQv0JOb5BTRybUcAFTdQPzutYqepY/U9hniWBqcDBefoK4NKq9WyQl+tT63fSHl/ug7GiHXPnaWUYxptJdzgXVd8F/SXgK5J+UiirDLu5Aml0vtmkISOANCaMpDebiGNDYEKkgbfqtREpob0gzTWsyaKk0fDIsTwa+dObPbIA2wBA0kakFswGpOEtKxtcmTQK4efWHREfSPonqTUG6bitlodimLPq/HNV4K0FjastcoKxogeAYaSBn96M2h2406umu5CGPvhjjXkns3CDGC3MMl1I//03JsVfNKMZ651Lvqp2N591iL9FOkV6kHTqVK8upE70X9aoa9agXm2JE4wVfRhp8KMF8TQweF7LSXqR9GHamNR/gqSVSUM1NrXOfpLWnEcr5hPSkJ9FfyclkBUiYl7DWL4A7ClJhVbMpk3EUctgUkI5LiJeB5D0jXnMuymp76WSmNbhs2FCnwbWXojj3a64k9ea6xRgP0mnKI2SN1jSXpLOBIiIfwF3Ab+V9FVJG5D6FmbMe5XcCzwG/CmPIPcFSdtJGpLrxwCL5bLlJC0eES+TOpsvz9tfJX+J7uhCAriI1Ll8rqQvStqL9LSCBTGONDznEXkbuwCnzmPeE3KMa5M6bz/hs4HAziCdWl4kaUNJq0naVdJvFzCeNs0JxpolIu4mPaPov0j9LI+ThrMcV5jtANJIevcBt5I+ZGOaWOds0kh9D5OGqnyRNMJe91w/mpQsriWdhh2TF/0u6UrSmaRHnNxGepzL2LzcONLVmh1JI+P9kMLQm3Xu72TS85yGkFpEJ5GucNVyLGl0/6eB1Umj0k3P63k2xzYI+FuOZyRzD9Td7nlEOzMrjVswZlYaJxgzK40TjJmVxgnGzErjBGNmpXGCMbPSOMGYWWmcYMysNP8PZqJJSQVOKgcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x252 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Evaluation on testing set (Using optimal threshold for group-wise performance)\n",
    "res = model_evaluation.model_evaluation_test(mdl, df_test, file_test, feature_col, scaler, thres=0.61)\n",
    "\n",
    "# Plot confusion matrix\n",
    "class_names = ['Severe', 'Moderate', 'Normal']\n",
    "df_cm = pd.DataFrame(\n",
    "    metrics.confusion_matrix(res['group_res']['true'], res['group_res']['pred']),\n",
    "    index=class_names, columns=class_names, \n",
    ")\n",
    "fig = plt.figure(figsize=(4, 3.5))\n",
    "heatmap = sns.heatmap(df_cm, annot=True, fmt=\"d\", cbar=False, cmap='Blues')\n",
    "heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(), rotation=0, fontsize=12)\n",
    "heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(), rotation=0, fontsize=12)\n",
    "plt.ylabel('True label', size=14)\n",
    "plt.xlabel('Predicted label', size=14)\n",
    "plt.tight_layout()\n",
    "plt.ylim([3, 0])\n",
    "# plt.savefig('../archive/Model_CM_group.png', dpi=300)\n",
    "\n",
    "# Print classification report\n",
    "print(metrics.classification_report(res['group_res']['true'], res['group_res']['pred']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T03:44:19.701202Z",
     "start_time": "2020-02-09T03:44:19.692197Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>true</th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>file</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>b09</th>\n",
       "      <td>Moderate</td>\n",
       "      <td>Moderate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a12</th>\n",
       "      <td>Severe</td>\n",
       "      <td>Severe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c07</th>\n",
       "      <td>Normal</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a09</th>\n",
       "      <td>Severe</td>\n",
       "      <td>Moderate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c04</th>\n",
       "      <td>Normal</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a13</th>\n",
       "      <td>Severe</td>\n",
       "      <td>Severe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a10</th>\n",
       "      <td>Moderate</td>\n",
       "      <td>Moderate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a06</th>\n",
       "      <td>Severe</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>b10</th>\n",
       "      <td>Moderate</td>\n",
       "      <td>Severe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c18</th>\n",
       "      <td>Normal</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a16</th>\n",
       "      <td>Severe</td>\n",
       "      <td>Severe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a38</th>\n",
       "      <td>Severe</td>\n",
       "      <td>Severe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c20</th>\n",
       "      <td>Normal</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a01</th>\n",
       "      <td>Severe</td>\n",
       "      <td>Severe</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          true      pred\n",
       "file                    \n",
       "b09   Moderate  Moderate\n",
       "a12     Severe    Severe\n",
       "c07     Normal    Normal\n",
       "a09     Severe  Moderate\n",
       "c04     Normal    Normal\n",
       "a13     Severe    Severe\n",
       "a10   Moderate  Moderate\n",
       "a06     Severe    Normal\n",
       "b10   Moderate    Severe\n",
       "c18     Normal    Normal\n",
       "a16     Severe    Severe\n",
       "a38     Severe    Severe\n",
       "c20     Normal    Normal\n",
       "a01     Severe    Severe"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res['group_res'].drop(['neg', 'pos'], axis=1).replace({'A': 'Severe', 'B': 'Moderate', 'C': 'Normal'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-07T04:13:48.729547Z",
     "start_time": "2020-02-07T04:13:48.350561Z"
    }
   },
   "source": [
    "## Minute-wise performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-07T04:13:48.729547Z",
     "start_time": "2020-02-07T04:13:48.350561Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.90      0.86      3988\n",
      "           1       0.83      0.71      0.76      2693\n",
      "\n",
      "    accuracy                           0.82      6681\n",
      "   macro avg       0.82      0.80      0.81      6681\n",
      "weighted avg       0.82      0.82      0.82      6681\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANUAAADCCAYAAADek4egAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAXDklEQVR4nO2de7QcVZWHv9+9ITwSCC4TEDEkDBIehkceRBBkgkYWziASZDQBnYkgTJiBKIhrVNBBR0BhZUAEBwIqgvISBMNDojKEZwIk4SEJDwVFcVSIRBR5JWHPH+d0KDrdVXVvV3dX3bs/Vq1UnT59aher9z3n7Nrnd2RmOI5THD3dNsBxBhruVI5TMO5UjlMw7lSOUzDuVI5TMO5UjlMwQ7ptgOP0l97NxpiteSm1jr307AIzO6BDJgHuVE6FsTUvs+GOM1LrvHz/N0Z2yJx1uFM51UVAT2+3rVgPdyqnwsidynEKR+q2BevhTuVUF3lP5TjFo/K9FXKnciqM91SOUyzC51SOUyyCnvL9hMtnkeP0hR7vqRynOPzlr+MUjTz65ziF4z2V4xSI5NE/xykc76kcp0jK+fK3fLM8x8mLCIGKtCOrCekASY9J+qWkzzb4fBtJt0q6X9JDkv4hq013KqfCxJ4q7Uj7ttQLnAe8H9gZmClp57pqJwNXmdkEYAbwzSyr3KmcatNaTzUF+KWZPWlmrwJXAB+sq2PAZvF8BPB/WY36nMqpLvmWfoyUtCRxPc/M5sXzrYHfJj57Gnhn3fdPAX4i6ThgGDAt64buVE61yQ6przSzyc2+3aCsfnOBmcDFZjZX0l7ApZLGm9lrzW7oTuVUFgE9PS3NYJ4GRieu38b6w7sjgQMAzGyRpI2AkcAzzRr1OZVTXZTjSOc+YHtJ20oaSghEzK+r8xvgvQCSdgI2Ap5Na9R7KqfCqKWeyszWSDoWWAD0At82s+WSvgwsMbP5wKeBCyUdTxgazrKM/afcqZxKoxbTlMzsJuCmurIvJs5XAHv3pU13Kqe6COTrqRynOIRa7qnagTuVU2lajP61hUHnVBqysWnopt02o3Am7LRNt01oG8uWLV1pZqPW+8CHf+VAQzdlwx0+3G0zCueue87ttgltY+MN9FSzz3z45zgFohZD6u3CncqpNuXrqNypnAojD1Q4TuH4nMpxCkTIo3+OUyjynspxCqeMc6ryWeQ4faG1pR95hF/OkvRAPB6X9OesNr2nciqL1Np7qoTwy/sICxbvkzQ/ZqYDYGbHJ+ofB0zIatd7KqfSSEo9Msgj/JJkJnB5VqPeUzmVJkf0r1Xhl3AfaQywLfC/WTd0p3KqS76Xv60Kv9SYAVxtZmuzbujDP6ey1HYnTTsyyCP8UmMGOYZ+4D2VU2lET2svf9cJvwC/IzjOYevdRdoBeBOwKE+j3lM5laaVQIWZrQFqwi+PEOSdl0v6sqSDElVnAldkCb7U8J7KqSwS9Pa2V/glXp/Slza71lNJMklzE9cnSjqlW/Y41aTFOVVb6Obw7xXgEEkju2iDU2UEPT1KPbpBN51qDTAPOL7+A0ljJN0S9wO6RdI2sfxiSedIulvSk5IO7bTRTnkI0b+WXv62hW4HKs4DDpc0oq78XOASM9sV+D5wTuKzrYB9gAOBr3bESqekpPdSg7Gnwsz+AlwCzKn7aC/gsnh+KcGJalxnZq/F/Kwt22+lU2a8p2rM2YSdFYal1EmGMl9JnAtA0qm1TOI22OeUFPmcqjFm9hxwFcGxatxNeBEHcDhwZ0YbJ5nZ7ma2e3usdMqKR/+aM5ew50+NOcDHJT0EfAz4ZFesckpPGXuqrr38NbPhifM/Apskrn8NvKfBd2Y1a8MZfNSGf2VjUGRUSDoaOBqADdwPBw6+QUHXiOtn5gH0bLJFrvwtpxp4T+U4RdLFYEQaTQMVkjZLOzpppOM0oraRdtqR2UaG8Eus82FJKyQtl3RZozpJ0nqq5YT3Q8m/BbVrAwbu3i1OZWilp8oj/CJpe+BzwN5mtkrSFlntNnUqMxvd7LO+IOk6wurKjYCvm9k8SS8AFwD7AauAGWb2rKSFwD2xfHPgSDO7Iz78V4GpwIbAeWZ2gaThwI8IC8g2AE42sx8VYbdTAVqP/q0TfgGQVBN+WZGocxTh97YKwMyeyWo013sqSTMkfT6ev03SpD4YfoSZTQImA3MkvZmQPbHMzCYCtwH/mag/xMymAJ9KlB8JPG9mewB7AEfF1ZovA9NjO/sBc1XGcJDTFmrbk2akKY2UtCRxHJ1oopHwy9Z1txkHjJN0l6TFkg7IsiszUCHpXEIvsC9wGvAicD7hx52HOZKmx/PRwPbAa8CVsex7wA8T9WvnS4Gx8Xx/YNdEVvqI2M7TwGmS9o1tbk3IB/xDTtucitOb3VO1KvwyhPBbm0rQsLhD0ngzayqqmSf69y4zmyjpfghpRZKG5vgekqYC04C9zOzFOLzbqEHVRrl9axP2CTjOzBbUtT8LGAVMMrPVkn7dpH1ngNLiuCSP8MvTwGIzWw38StJjBCe7r1mjeYZ/qyX1EH/4cfj2Wk6jRwCrokPtCOyZuG+t1zmMjNw+gobAMZI2iDaMkzQstv9MdKj9gDE57XIGAFLoqdKODNYJv8SOYgYwv67OdYSpBXFB7TjgybRG8/RU5wHXAKMkfQn4MPClHN8DuBmYHXP4HgMWx/K/Ae+QtBR4HvhIRjsXEYaCy+Kc6VngYMJaq+sVxBIfAB7NaZczQGhlCm1mayTVhF96gW/XhF+AJWY2P362v6QVhNHTZ8zsT6k25RGIkfQOwjAO4BYze7jfTxLae6FbeXs9m2xhA3Ej7VX3DeiNtJc2mhdtPmYn2+fzl6R+98bZUxp+t53kzajoBVYThoBlyWx3BjvKNcTrOJkOIukkgjLnWwkTucskfa6Vm3p2uVMUZVxPlaen+ighuvYihFW2hHD36e00zHGyELlC6h0nj1M9VVdvCBnRD8fpFGV819/UqSSdRZhDvQgsl7QgXu9PdgjccdpOLaReNtJ6qlqEbzlwY6J8cYO6jtMVyudS6Qm13+qkIY7TVyo7p5K0HXAqsDOJFCAzG9dGuxwnmy5q+6WR553TxcB3CH8Y3k+QE7uijTY5Tm7KqKaUx6k2qSWymtkTZnYyMRfKcbqJgB6lH90gT0j9lZhv94Sk2YQd5zJXPzpOJ+ip6PDveGA4QeByb8JKyCPaaZTj5EEKTpV2ZLeRrlEhaZakZ2uy4pI+kdVmZk9lZvfE078S1GIdpzS0Mm/Ko1ERudLMjs3bbtrL32tZfxXkOszskLw3cZx2IFpOqM2jUdFn0nqqAbmWYIfttubiq7/SbTMKZ4cTru+2CZ2n9aTZRhoV72xQ70NRsuFx4Hgz+22DOutIe/l7S3+sdJxO0pvtVSPjItYa86JiMeTTqLgeuNzMXomBuu/SQOc/iSvUOpWltj1pBmnCL5kaFXWrfC8EvpZ1Q19w6FSaIT3pRwaZGhWStkpcHgQ8kmlTXuMlbWhmr2TXdJzOEBYitl2jYo6kgwgbvz8HzMpqN0/u3xTgWwTlom0k7QZ8wsyO6/fTOE5B9LY41jKzm4Cb6sq+mDj/HEH2OTd5TDqHsBP8n+JNHsTTlJwSENKUWnv52w7yDP96zOypum52bZvscZw+0Vu+LKVcTvXbOAS0+Ab6OEK83nG6irrYG6WRx6mOIQwBtwH+CPwsljlO12l1TtUO8uT+PUMINTpOqajNqcpGnujfhTTIATSzoxtUd5zOoYr2VIThXo2NgOm8MV/KcbqCyJWm1HHyDP+uTF5LuhT4adsscpw+UELdl37l/m2Lb1njlIAqqymt4vU5VQ8hVaPhLt6O01G6qJeeRqpTRW2K3Qi6FACvWZ69dxynAwgYUsKeKjV2Eh3oWjNbGw93KKdUlHHXjzwByXslTWy7JY7TR4ToVfqR2UaG8Eui3qGSTFLmBnJpGhVDzGwNsA9wlKQnCNuKitCJuaM53aVFbb+8wi+SNiWoid2zfivrkzanuheYSNhb13FKRwHRv7zCL/8FnAGcmKfRNKcSBFXaPpvqOB2ixTSlTOEXSROA0WZ2g6SWnWqUpBOafWhm/53nBo7TLkJGRWa1fgu/SOoBziLHat8kaU7VS1CmLV/M0nEgvqdqq/DLpsB4YGG8z1uA+ZIOMrOko76BNKf6vZl9Octix+kWBeT+rRN+IbyLnQEcVvvQzJ4HRq67n7QQODHNoSA9pN5yDyVpegxD7thqW47TiFZ2/YjR7ZrwyyPAVTXhlyj20i/Seqr39rfRBDMJ+wPPAE4poD3HSdD6pm9Zwi915VPztNm0pzKz5/piXD2ShhN2CTmSuMhR0lRJt0u6VtIKSefHySCSXpB0qqQHJS2WtGUsHyXpGkn3xWPvWD5F0t2S7o//7tCKvU71qA3/Wnn52w7aucTrYOBmM3sceC6RlTEF+DSwC7AdUNvoYBiw2Mx2A24nbNkD8HXgLDPbA/gQcFEsfxTY18wmAF8ETmtmiKSjJS2RtOTPz60s7AGd7qOMoxu0U/Z5JnB2PL8iXt8I3Jt42XY5IWPjauBV4IZYfynhLTfANGDnRDe/WXzDPQL4rqTtCWHQDZoZEkOo8wB22mWC5y8OEKSKLlLsD5LeTBBxHy/JCOF5I4xd63/UtevViYTdtQnbeoC9zOylunt8A7jVzKZLGgssLPgxnApQ1Y20+8OhwCVmNsbMxprZaOBXhF5pStSu7gE+QghkpPETQoQGAEm7x9MRvL4kZVaRxjvVoYx7/rbLqWYC19aVXUN4B7AI+CrwMMHR6uvVMweYLOkhSSuA2bH8DOB0SXcRekJnkCGgB6Ue3aAtw79GoUczO0fSQ4SXZx9p8PnwxPnVhHkWZraS0KPV118EjEsUfaF1y51qUV0xTccpLSX0qc46lZktxAMKTkEMquif43SKEvqUO5VTXSorpuk4ZaaMgYoSKlE7Tn6U8V/m9zOEXyTNlvRzSQ9IulPSzlltulM5laVVNaWE8Mv7gZ2BmQ2c5jIz28XMdie8G81c8e5O5VSXDM2/HCPDdcIvZvYqIUf1g8kKZvaXxOUwGuyAU4/PqZzKUkCgIlP4BUDSvwMnAEMJOa2peE/lVJocSz9G1pb9xOPouq/X02gvtvPMbDvgP4CTs2zynsqpNG0WfqnnCuB/sm7oPZVTaVqcU60TfpE0lLBCff4b29f2ict/BH6R1aj3VE6laWVKZWZrJNWEX3qBb9eEX4AlZjYfOFbSNGA1sAr4l6x23amcyhLmTe0VfjGzT/a1TXcqp7p0cSFiGu5UToVpXaKsHbhTOZWmhD7lTuVUF+FO5TiF02qgoh24UzmVxgMVjlMk3ZShTWHQOdWjDz+wcs+3v+mpDt1uJDAQdaY7/VxjGhWKci5SHHROZWajOnUvSUtS8s4qS5meq3wuNQidyhlY+HsqxymYEvqUO1WbmZddpZKU5rnK6FS+9KONJHZBH1CU5blqCbVtFn45IW5Q+JCkWyQ1DJokcadyqkvGjh9Z77ByCr/cD0w2s10J+v5nZJnlTuVUmJBQm3ZkkEf45VYzezFeLiasDk7FnaqPSDJJcxPXJ0o6pYsmtYSk6fGZduy2Lf2hxZW/jYRftk6pfyTw46xG3an6zivAIZJGdtuQgphJ2HhvRrcN6Su1hNoMp2pZ+AVA0keBycCZWXa5U/WdNYTo1/H1H0gaEyeztUntNrH8YknnSLpb0pOSDu200Y2QNBzYm/AXeEYsmyrpdknXxgn6+XHXSyS9IOlUSQ9KWixpy1g+StI1ku6Lx96xfEp85vvjvzsU/gzZgYqVZjY5cSSDLLmEX+Jy+pOAg8zslSyb3Kn6x3nA4ZJG1JWfS9iWdVfg+8A5ic+2ImzPeiBhJ8kycDBws5k9DjwnaWIsnwJ8GtgF2A44JJYPAxab2W7A7cBRsfzrwFlmtgfwIeCiWP4osK+ZTQC+CJxW9AO0uD1pHuGXCcAFBId6Jo9N/p6qH5jZXyRdQtg6NbnB9168/gO8lDdGiq4zs9eAFbW/8CVgJnB2PL8iXt8I3GtmTwJIupzwx+Bq4FXghlh/KfC+eD4N2DkRGNhM0qaEfZm/GxWJDNigUOvzzZuaklP45UxgOPCD+Hy/MbOD0tp1p+o/ZwPLgO+k1EmOz5PDBgFIOpUge0XU6u4Ykt5MUFsdL8kIPyojiKDUzytq16vNrHa+ltd/Pz3AXmaW/AODpG8At5rZdEljKXjDvzCnarvwy7S+tunDv35iZs8BVxHmIzXu5vUJ/+GEAEBaGyeZ2e6ddqjIoYSh6hgzG2tmowkbm+8DTIlDoh7CfsupzwH8BDi2diGp9jwjgN/F81lFGr/uXhlHN3Cnao25hGUQNeYAH48bhn8M6LO8VQeZCVxbV3YNcBiwiDDve5jgaPX16pkDTI4BmhXA7Fh+BnC6pLsIPWHh9EipRzfQ672544ToH3CimR3YbVuy2G3CJFtw2+LUOluNGLq008tUfE7lVBa57p9TBcxsIQUHFNqJC784TsGUcemHO5VTYboXjEjDo385kHSdpKWSltdyx2LKzlxJy2JK0qhYvlDS1yTdK+lxSe+O5b2SzoxpPA9J+tdYPjx+f5nChs0fbG6JkyRn7l/HcafKxxFmNomQUDknvjgdBiwzs4nAbcB/JuoPMbMpwKcS5UcCz8dUnj2AoyRtC7wMTI/t7AfMVRmFF0pKGZ3Kh3/5mCNpejwfDWwPvAZcGcu+B/wwUb92vhQYG8/3B3ZNJNOOiO08DZwmad/Y5tbAlsAfin+MgYcHKipIfG8zjZCG86KkhcBGDao2SklKpvIIOM7MFtS1PwsYBUwys9WSft2kfaeOsobUffiXzQhgVXSoHYE9Y3kPIdUHQhZCVirPAuAYSRsASBonaVhs/5noUPvRRDjSaUIJ85S8p8rmZmB2TD16jLCkGuBvwDskLQWeJ+TIpXERYSi4LM6ZniUsvfg+cL2kJcADhOUSTk7KGP3zNKV+IukFMxvebTsGMxMnTbY7F92XWmfYhj2epuQ4faGMgVLvqZzKIulm3rhKoBErzeyATthTw53KcQrGo3+OUzDuVI5TMO5U/UTSWkkPSHpY0g8kbdJCW1Ml3RDPD1IDTe9E3c0l/Vs/7nGKpBPzltfVubgvsmqSxkp6uK82DhTcqfrPS1FfYjxBZWh28kMF+vz/18zmm1mahNnmQJ+dyukc7lTFcAfw9vgX+hFJ3yQoLY2WtL+kRTEL/QcKApa13SYelXQnr8uaIWmWpHPj+ZYKopYPxuNdBO2I7WIveWas95lE9vuXEm2dpLCjxc+ATCFLSUfFdh5UEMdM9r7TJN0RM+8PjPUbZt4PdtypWkTSEMKuET+PRTsQVIomELIuTgamxSz0JcAJkjYCLgQ+ALwbeEuT5s8BbovilROB5cBngSdiL/kZSfsTEnOnALsDkyTtK2kSQdlpAsFp98jxOD80sz3i/R7hjUpRY4G/J0iqnR+foVnm/aDGX/72n40lPRDP7wC+BbwVeMrMaqlMexK2aLkrvqQcSlAq2hH4lZn9AkDS94CkxneN9wD/DGBma4HnJb2prs7+8bg/Xg8nONmmwLW1HSskzSeb8ZK+QhhiDifkK9a4KoqB/kLSk/EZmmXeP57jXgMWd6r+81K9Xl90nL8li4CfmtnMunq700QIvx8ION3MLqi7x6f6cY+LgYPN7MGYPT818Vkjgc1mmfdj+3jfAYUP/9rLYmBvSW8HkLSJpHGEpNltJW0X681s8v1bgGPid3slbQb8ldAL1VgAHJGYq20taQuC1vl0SRsrSDB/IIe9mwK/j5n0h9d99k+SeqLNf0dILm6WeT+o8Z6qjZjZs/Ev/uWSNozFJ5vZ4wrL8m+UtJKwbGR8gyY+CcyTdCRhbdYxZrZI0l0xZP3jOK/aCVgUe8oXgI+a2TJJVxIy358iDFGz+AJwT6z/c97ovI8RVjhvCcw2s5clNcu8H9R4mpLjFIwP/xynYNypHKdg3Kkcp2DcqRynYNypHKdg3Kkcp2DcqRynYNypHKdg/h8M5N3NjaOq6gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 216x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "disp = metrics.plot_confusion_matrix(\n",
    "    mdl, scaler.transform(df_test[feature_col]), df_test['apn'],\n",
    "    display_labels=['Non-\\napnea', 'Apnea'],\n",
    "    cmap=plt.cm.Blues, include_values=False,\n",
    "    normalize='true')\n",
    "disp.figure_.set_size_inches((3, 3))\n",
    "plt.tight_layout()\n",
    "plt.savefig('../archive/Model_CM_minute.png', dpi=600)\n",
    "\n",
    "print(metrics.classification_report(df_test['apn'], mdl.predict(scaler.transform(df_test[feature_col]))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Comparison of testing performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-07T17:57:36.738103Z",
     "start_time": "2020-02-07T17:57:23.019225Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Logistic Regression ***\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.90      0.86      3988\n",
      "           1       0.83      0.71      0.76      2693\n",
      "\n",
      "    accuracy                           0.82      6681\n",
      "   macro avg       0.82      0.80      0.81      6681\n",
      "weighted avg       0.82      0.82      0.82      6681\n",
      "\n",
      "*** MLP ***\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.89      0.86      3988\n",
      "           1       0.82      0.73      0.77      2693\n",
      "\n",
      "    accuracy                           0.83      6681\n",
      "   macro avg       0.82      0.81      0.82      6681\n",
      "weighted avg       0.83      0.83      0.82      6681\n",
      "\n",
      "*** Light GBM ***\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.88      0.85      3988\n",
      "           1       0.80      0.71      0.75      2693\n",
      "\n",
      "    accuracy                           0.81      6681\n",
      "   macro avg       0.81      0.79      0.80      6681\n",
      "weighted avg       0.81      0.81      0.81      6681\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Define and CV evaluate models with the optimal hyperparameters\n",
    "logreg = LogisticRegression(C=0.05, max_iter=1e4)\n",
    "mlp = MLPClassifier(\n",
    "    hidden_layer_sizes=(32, ) * 6, \n",
    "    max_iter=2000, early_stopping=True, validation_fraction=0.1,\n",
    "    random_state=123)\n",
    "gbm = LGBMClassifier(\n",
    "    boosting_type='gbdt', num_leaves=8, max_depth=8, \n",
    "    learning_rate=0.1, n_estimators=50, \n",
    "    random_state=123, n_jobs=-1, silent=True)\n",
    "\n",
    "for mdl_name, mdl in zip(['Logistic Regression', 'MLP', 'Light GBM'], [logreg, mlp, gbm]):\n",
    "    scaler = preprocessing.StandardScaler().fit(df_train[feature_col])\n",
    "    mdl.fit(scaler.transform(df_train[feature_col]), df_train['apn'])\n",
    "    print(f'*** {mdl_name} ***')\n",
    "    print(metrics.classification_report(\n",
    "        df_test['apn'],\n",
    "        mdl.predict(scaler.transform(df_test[feature_col]))\n",
    "         ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-07T18:27:29.602545Z",
     "start_time": "2020-02-07T18:27:29.188649Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARgAAADQCAYAAADcQn7hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAWVUlEQVR4nO3de5hVdb3H8fdHNIFAPIgZkTGKlOINZdDMa2k+nm7aOVoQpqZJWpllVKR1HrudQ1ne6xSapYaGZhwp8xZH8sZtuA1E5SWxyE5lKoli4vQ9f6zfyGa3Z2YPs397z8Dn9TzzsPZv/dZa370f5jO/tfa6KCIwM8thm0YXYGZbLgeMmWXjgDGzbBwwZpaNA8bMstm20QX0FcOGDYumpqZGl2HWKy1evPjJiNi5vN0BU6WmpiZaWloaXYZZryTp8Urt3kUys2wcMGaWjQPGzLJxwJhZNg4YM8vGAWNm2ThgzCwbB4yZZeOAMbNsHDBmlo0DxsyyccCYWTa+2LFaixeD1OgqzGqnDvfj9gjGzLJpWMBIWleDdbxG0o86mb+jpA9X29/MaqtPj2Ai4omIOLGTLjsCH+5GfzOroV4VMJJGSpojqTX9+7rUPkrSfEmLJH2xffQjqUnSyjS9t6SFkpal5UcD04BRqe2isv79JH1d0orU/5xGvW+zLVWvChjgSuC6iNgPmAFcntovAy6LiPHAEx0se1bqMxZoBtYAU4FHI2JsRHyqrP9kYDfggJLtmVkN9baAOQS4IU1fDxxW0n5zmr6hfKFkHnC+pM8AIyNifRfbOgb4dkS8BBART2121WZWUW8LmHJVf48WETcA7wLWA3dKeksXi6g76zez7uttAfMgMCFNTwLuT9PzgX9P0xPKFwKQtDvw24i4HJgN7Ac8CwzuYFt3AWdJ2jYtP7TH1ZvZJhoZMAMlrSn5OQ/4GPABSa3A+4FzU9+PA+dJWggMB9ZWWN97gZWSlgF7UhzL+SvwgKSVki4q63818DugVdJy4H01f4dmWzlFHc7m6ylJA4H1ERGSJgATI+L4Omx3MsXBYF4H4yo+l8Gsr6rh776kxRHRXN7eVy4VGAdcKUnAM8Dp9dhoREwHpgM0S70/ic16mT4RMBFxH7B/o+sws+7ZrGMwktrSyWsrJf1E0o61LqwnJP2st9VktjXa3IO869PJa/sATwEfqUUx7d/o9FREvC0inqnFusxs89XiW6R5wIj2F5I+lU7pb5X0hZL2z0v6taS7Jd0oaUpqnyvpPyX9AjhX0s6SbknrWCTp0NTvyDRqWiZpqaTBkoZLurdkNHV46rta0rA0fV6at1LSx1Nbk6RfSbpK0i8l3SVpQA0+CzMr0aMRg6R+wNHAd9PrY4HRwEEUJ7LNlnQE8DzFeSwHpG0uARaXrGrHiDgyreMG4JKIuD9di3QnsBcwBfhIRDwgaRDwAsU3PHdGxFdSLQPL6hsHfAA4ONWzIAXZ06nOiRFxpqSbUn0/6MnnYWab2tyAGZDON2miCIq7U/ux6Wdpej2I4hd5MHBr++n7kn5Str6ZJdPHAGO08eZOO0gaDDwAXCxpBvDjiFgjaRFwjaTtgP+JiGVl6z0MmBURz6Xt/hg4nOJEvMdK+i9O78XMaqhHx2CAkcAr2HgMRsB/peMzYyNij4j4bmrvzHNlNR1Sso4REfFsREwDPggMAOZL2jMi7gWOAP4AXC/plLL1drbdv5dMt9FHvlEz60t6dAwmItZSnH07JY0i7gROT7swSBoh6VUUp/y/U1L/NO/tnaz2LuCj7S8kjU3/joqIFRHxVaAF2FPSSODPEXEVxW7agWXruhc4QdJASa8E3g3ct1lvdty44sQk//hnS/mpgx7/1Y6IpelU+wkRcb2kvYB5aRdnHXByRCySNBtYDjxOERCVTveHIrC+mS4X2JYiJM4CPi7pzRSjjVXA7RTXJX1K0oa0rU1GMBGxRNL3gYWp6epUb1NP37eZda1ulwpIGhQR69Jp//cCkyNiSV02XgPNzc3R0tLS6DLMeqXecKnAdEljgP7AtX0pXMxs89QtYCLCVyubbWV62/1gzGwL4oAxs2wcMGaWjQPGzLJxwJhZNg4YM8vGAWNm2ThgzCwbB4yZZeOAMbNsHDBmlo0DxsyyccCYWTYOGDPLxgFjZtn4RtdVWvGHtTRNva3RZZjV1eppnd0+u2s1G8FICknXl7zeVtJfJP20imXXpX+bJL2vpL1Z0uVp+jRJV9aqXjPLr5a7SM8B+5Q8IfGtFI8T6Y4m4OWAiYiWiPhYbcrrXK0eW2tmG9X6GMztbHwkyUTgxvYZki5sf1xser2ywt39pwGHp0fBfkLSUZVGQJLeKWlBeoTszyXtImkbSQ9L2jn12UbSI5KGdfI42gslTZd0F3BdLT8IM6t9wPwQmCCpP7AfsKCby08F7ksPXLukk373A2+MiAPSNj8dEf+gePTrpNTnGGB5RDwJXEbxONrxFI+IvbpkXeOA433PYLPaq+luQUS0plHJROBntVx3mdcCMyUNp3iy5GOp/RrgVuBS4HTge6m9o8fRAsxuf6StmdVWjq+pZwNfp2T3KHmpbHv9e7CNK4ArI2Jf4EPt64qI3wN/kvQWigfe3576V3wcbZr3HGaWRY6AuQb4YkSsKGtfTXq0q6QDgd0qLPssMLhCe7khbDyAfGrZvKspdpVuioi21FbxcbRmllfNAyYi1kTEZRVm3QIMlbQMOBt4qEKfVuAlScslfaKTzVwI3CzpPuDJsnmzgUFs3D2C4nG0zZJaJa2ieBStmWVWt0fH1oukZooDuofXcr3bDx8dw0+9tJarNOv1qj3Rrjc8OjY7SVMpRkeTuupb5fomA5MB+u2wcy1WabZV2aKuRYqIaRExMiLur9H6pkdEc0Q09xs4pBarNNuqbFEBY2a9y2YHjKS2dMbtSkk3SxpYy8K6Ucf5jdiumXWtJyOY9emckn2AF+nGNzOS+vVgu+UqBowKHqGZNVCtfgHvA/YAkHSypIVpdPOd9jCRtE7SFyUtAA6RNF7Sg+kr6YWSBkvqJ+midL1Qq6QPpWWPknSvpFmSVkn6drrWaBowIG1rRroa+1eSvgUsAXaVNFHSijTS+mp7wamer6Ttz5e0S40+CzNLehww6SrkfwVWSNoLeC9waESMBdrY+I3OK4GVEXEwsBCYCZwbEftTnMq/HjgDWJuuGRoPnCmp/YS8g4BPAvsCo4B/i4ipbBxJtW/nDcB16TqlDcBXgbcAY4Hxkk4oqWd+2v69wJk9/SzMbFM9CZgB6aS5FuB3wHeBoykuHlyU5h0N7J76t1GcbAdFCPwxIhYBRMTfIuIl4FjglLTsAmAnYHRaZmFE/DadnXsjcFgHdT0eEfPT9HhgbkT8Ja1/BnBEmvci0H6l9mKKW0WYWQ315DyY9WmU8jIVVxNeGxGfrdD/hZJT9wVUOsNPwDkRcWfZeo+q0L+jMwRLry1SB30ANsTGswzb2MLOCTLrDWr9SzUHuFXSJRHxZ0lDgcER8XhZv18Dr5E0PiIWpSub1wN3AmdL+t+I2CDp9Wy85uigtLv0OMVu2PTUvkHSdhGxoUI9C4DLJA0Dnqa4yvuKzXlj+44YQksPbx9otrWp6bcsEbEK+Bxwl6RW4G5geIV+L1KExBWSlqd+/SkuVFwFLJG0EvgOG0NwHsUNqVZS3J5hVmqfDrRKmlFhO38EPgvcAywHlkTErbV5t2bWlT5xLVLaRZoSEe9oVA3Nzc3R0tLSqM2b9WodXYvk80TMLJs+cWAzIuYCcxtchpl1k0cwZpaNA8bMsnHAmFk2Dhgzy8YBY2bZOGDMLBsHjJll44Axs2wcMGaWjQPGzLJxwJhZNg4YM8vGAWNm2ThgzCwbB4yZZdMn7gfTG6z4w1qapt7W6DLM6mJ1je4/7RGMmWXjgDGzbBwwZpaNA8bMsnHAmFk2NQkYSesqtJ0l6ZQuljtN0pUdzDu/k+UGSfpvSY9KWippsaQz07wmSeslLZO0XNKDkt6Q5h0lKSSdUbKuA1LblGrfr5lVJ9sIJiK+HRHX9WAVHQYMxRMgnwZGR8QBwHHA0JL5j0bE2IjYH7i2bF0rKJ4q2W4CxVMfzazGsgWMpAvbRwWSxktqlTRP0kXpsbDtXiPpDkkPS/pa6j8NGJBGITPK1jsKOAj4XET8AyAi/hIRX+2glB0owqjd74D+knaRJIpwur0W79nMNlWvE+2+B0yOiAdTeJQaCxwA/B34jaQrImKqpI9GxNgK69obWN4eLh0YJWkZMBgYCBxcNv9HwEnAUmBJ2raZ1Vj2g7ySdgQGR8SDqemGsi5zImJtRLxA8eD7kd1c/wVppPNESXP7LtIo4OPA9LLFbqIImInAjZ2se7KkFkktbc+v7U5ZZkZ9vkVSF/NLRw9tdD2qWgXsL2kbgIj4Shrp7NBB/9nAEaUNEfF/wAbgrcCcjjYUEdMjojkimvsNHNJFWWZWLnvARMTTwLOS3piaJlS56AZJ21VY3yNAC/BlSf0AJPWn4yA7DHi0Qvt/AJ+JiLYq6zGzbqrVMZiBktaUvL64bP4ZwFWSnqN4iH01+xvTgVZJSyJiUtm8DwIXAY9IegpYD3ymZH77MRgBL6b+myjZZTOzTBQR+TciDYqIdWl6KjA8Is7NvuEa2n746Bh+6qWNLsOsLrp7NbWkxRHRXN5er2+R3i7ps2l7jwOn1Wm7ZtZAdQmYiJgJzKzHtsys9/C1SGaWjQPGzLLxLTOrtO+IIbTU6DaCZlsLj2DMLBsHjJll44Axs2wcMGaWjQPGzLJxwJhZNg4YM8vGAWNm2ThgzCwbB4yZZeOAMbNsHDBmlo0DxsyyccCYWTYOGDPLxgFjZtk4YMwsGweMmWXjgDGzbHxP3iqt+MNamqbe1ugyzLLq7gPXuuIRjJll44Axs2wcMGaWjQPGzLJxwJhZNlUFjKQLJP1SUqukZZIOzl1Yd0m6UNLzkl5V0rauiuXOz1uZ2dary4CRdAjwDuDAiNgPOAb4fe7CNtOTwCe7uYwDxiyTakYww4EnI+LvABHxZEQ8Iek4Sb+WdL+kyyX9FF4eSUxpX1jSSklNafpkSQvTKOg7kvql9mMlzZO0RNLNkgZJak79lklaISlS31GS7pC0WNJ9kvYsqfUa4L2Shpa/iUrbljQNGJDaZmzWJ2hmHaomYO4CdpX0kKRvSTpSUn/gKuCdwOHAq7taiaS9gPcCh0bEWKANmCRpGPA54JiIOBBoAc6LiJaIGJv63gF8Pa1qOnBORIwDpgDfKtnMOoqQObeabUfEVGB92s6kCjVPltQiqaXt+bVVfFRmVqrLM3kjYp2kcRRB8mZgJjANeCwiHgaQ9ANgcherOhoYByySBDAA+DPwRmAM8EBqfwUwr30hSe8BDgSOlTQIeBNwc+oLsH3Zdi4Hlkn6RhXb7uq9T6cINLYfPjq66m9mm6rqUoGIaAPmAnMlrQBOBTr6hXuJTUdG/dO/Aq6NiM+Wdpb0TuDuiJhYviJJewNfAI6IiDZJ2wDPpFFIR7U+I+kG4MOlq6q0bTPLq5qDvG+QNLqkaSzwJ2A3SaNSW2k4rKYYcSDpQGC31D4HOLH9Wx5JQyWNBOYDh0raI7UPlPR6SUOAHwKnRMRfACLib8Bjkk5KfSVp/wplXwx8iI0B2tG2ATZI2q6rz8HMuq+aYzCDgGslrZLUSrE7M5Vil+g2SfcDj5f0vwUYKmkZcDbwEEBErKI41nJXWs/dwPAUHqcBN6b2+cCewAnASOCq9oO9af2TgDMkLQd+CRxfXnBEPAnMIu0+dbTt1H060OqDvGa1p4ieH1qQdBQwJSLe0eOV9VLbDx8dw0+9tNFlmGW1uVdTS1ocEc3l7T6T18yyqcn9YCJiLsVBYDOzl3kEY2bZOGDMLBvfMrNK+44YQkuNbydotqXzCMbMsnHAmFk2Dhgzy8YBY2bZOGDMLBsHjJll44Axs2wcMGaWjQPGzLJxwJhZNg4YM8vGAWNm2ThgzCybmtwyc2sg6VngN42uoxuGUTzpsi/pazW73o1GRsTO5Y2+XUP1flPpnqO9laSWvlQv9L2aXW/XvItkZtk4YMwsGwdM9aY3uoBu6mv1Qt+r2fV2wQd5zSwbj2DMLBsHjJll44ApI+k4Sb+R9IikqRXmby9pZpq/QFJT/avcpJ6u6j2v/bnikuZIGtmIOkvq6bTekn4nSgpJDf0auJp6Jb0nfca/lHRDvWusUE9X/ydeJ+keSUvT/4u3ZSsmIvyTfoB+wKPA7sArgOXAmLI+Hwa+naYnADN7eb1vBgam6bN7e72p32DgXmA+0Nyb6wVGA0uBf0mvX9WoertR83Tg7DQ9Blidqx6PYDZ1EPBIRPw2Il4EfggcX9bneODaNP0j4GhJqmONpbqsNyLuiYjn08v5wGvrXGOpaj5fgC8BXwNeqGdxFVRT75nANyPiaYCI+HOdayxXTc0B7JCmhwBP5CrGAbOpEcDvS16vSW0V+0TES8BaYKe6VPfPqqm31BnA7Vkr6lyX9Uo6ANg1In5az8I6UM3n+3rg9ZIekDRf0nF1q66yamq+EDhZ0hrgZ8A5uYrxpQKbqjQSKf8ev5o+9VJ1LZJOBpqBI7NW1LlO65W0DXAJcFq9CupCNZ/vthS7SUdRjA7vk7RPRDyTubaOVFPzROD7EfENSYcA16ea/1HrYjyC2dQaYNeS16/ln4ePL/eRtC3FEPOpulT3z6qpF0nHABcA74qIv9eptkq6qncwsA8wV9Jq4I3A7AYe6K32/8OtEbEhIh6juCB2dJ3qq6Sams8AbgKIiHlAf4oLIWuvkQeketsPxV+j3wK7sfEA2d5lfT7Cpgd5b+rl9R5AcdBvdF/4fMv6z6WxB3mr+XyPA65N08Modk926uU13w6clqb3ogggZamn0f/petsP8DbgofRLeUFq+yLFX38o0v5m4BFgIbB7L6/358CfgGXpZ3Zvrresb0MDpsrPV8DFwCpgBTChkfVWWfMY4IEUPsuAY3PV4ksFzCwbH4Mxs2wcMGaWjQPGzLJxwJhZNg4YM8vGAWPZSXp3ujJ6z0bXYvXlgLF6mAjcT3FiYhaS+uVat20+B4xlJWkQcCjF6ekTSto/LWmFpOWSpqW2PST9PLUtkTRK0lGSflqy3JWSTkvTqyX9h6T7gZMknSlpUVr+FkkDU79dJM1K7cslvUnSlySdW7Ler0j6WF0+lK2IL3a03E4A7oiIhyQ9JelAYJfUfnBEPC9paOo7A5gWEbMk9af4A7hr5dW+7IWIOAxA0k4RcVWa/jJFqF0BXA78IiLenUY6gyhOj/8xcFm6yHICxa0OrIYcMJbbRODSNP3D9Hob4HuR7lMTEU9JGgyMiIhZqe0FgCputTOzZHqfFCw7UoTInan9LcApab1tFLfYWCvpr+n2ELsASyPirz15o/bPHDCWjaSdKH6595EUFHdbC+AWqrsNBsBLbLor379s/nMl098HToiI5Wk36qguSrya4tYQrwau6aKvbQYfg7GcTgSui4iREdEUEbsCj1Hc3uL0kmMkQyPib8AaSSektu3T/MeBMen1EODoTrY3GPijpO2ASSXtcyhuF4qkfpLa7+Y2i+Jq6PFsHO1YDTlgLKeJFL/EpW4BXgPMBlokLQOmpHnvBz4mqRV4EHh1RPye4t4lrRTHaJZ2sr3PAwuAu4Ffl7SfC7xZ0gpgMbA3QBS3lLyH4pYbbZv7Jq1jvpratlrp4O4S4KSIeLjR9WyJPIKxrZKkMRT39JnjcMnHIxgzy8YjGDPLxgFjZtk4YMwsGweMmWXjgDGzbP4fXLRt9mFA+5wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(4, 3))\n",
    "plt.barh(np.arange(3, 0, -1), [0.86, 0.85, 0.84], height=0.5)\n",
    "plt.barh([4], [0.82], height=0.5, color='r')\n",
    "plt.yticks(\n",
    "    [4, 3, 2, 1], \n",
    "    labels=[\n",
    "        'Logistic\\nRegression', \n",
    "        'Multilayer\\nPerceptron',\n",
    "        'Light GBM',\n",
    "        'SqueezeNet'],\n",
    "    )\n",
    "plt.ylim([0.4, 4.6])\n",
    "plt.xlabel('Accuracy')\n",
    "plt.tight_layout()\n",
    "plt.savefig('../archive/Model_Minute_Accuracy.png', dpi=600)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Backup block for sample visualization\n",
    "\n",
    "\n",
    "file = 'a15'\n",
    "with open('../HR_data/' + file + '.pkl', 'rb') as f:\n",
    "    data = pickle.load(f)\n",
    "\n",
    "b, a = signal.butter(3, 0.1)\n",
    "t, hr = data['t'], data['hr']\n",
    "idx_valid = (hr < 2) & (hr > 0.5)\n",
    "hr, t = hr[idx_valid], t[idx_valid]\n",
    "hr_smth = signal.filtfilt(b, a, hr)\n",
    "\n",
    "res = np.transpose(seg_pred[file])\n",
    "assert res.shape[0] == np.ceil(t[-1]) - 4\n",
    "\n",
    "select_num = 3\n",
    "\n",
    "plt.figure(figsize=(16, 12))\n",
    "for case in range(4):\n",
    "    if case == 0:\n",
    "        val = [1, 0]\n",
    "        label = 'False Negative'\n",
    "    elif case == 1:\n",
    "        val = [0, 1]\n",
    "        label = 'False Positive'\n",
    "    elif case == 2:\n",
    "        val = [1, 1]\n",
    "        label = 'True Positive'\n",
    "    elif case == 3:\n",
    "        val = [0, 0]\n",
    "        label = 'True Negative'\n",
    "        \n",
    "    idx = np.where((res == val).all(axis=1))[0]\n",
    "    idx_plot = np.random.choice(idx, select_num)\n",
    "    \n",
    "    plt.subplot(2, 2, case + 1)\n",
    "    for idx in idx_plot:\n",
    "        idx_t = (t > idx) & (t < idx + 5)\n",
    "        plt.plot(t[idx_t] - idx, hr_smth[idx_t])\n",
    "        \n",
    "    plt.title(label)\n",
    "    plt.xlabel('minute')\n",
    "    plt.ylabel('HR')\n",
    "    \n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
